Merge branch 'tristan-erp5-cluster' into erp5-cluster

Conflicts:
	stack/erp5/buildout.cfg
	stack/erp5/instance-zope.cfg.in
	stack/erp5/instance.cfg.in
	stack/monitor/buildout.cfg

component/pycurl/buildout.cfg

+[buildout]
+extends =
+  ../curl/buildout.cfg
+  ../openssl/buildout.cfg
+
+parts =
+  pycurl
+
+[pycurl-env]
+PATH = ${curl:location}/bin:${openssl:location}/bin:%(PATH)s
+PYCURL_SSL_LIBRARY=openssl
+CPPFLAGS=-I${openssl:location}/include
+CFLAGS=-I${openssl:location}/include
+
+[pycurl]
+recipe = zc.recipe.egg:custom
+egg = pycurl
+
+rpath =
+  ${curl:location}/lib/
+  ${openssl:location}/lib/
+environment = pycurl-env

slapos/recipe/generic_mysql/template/my.cnf.in

@@ -34,7 +34,7 @@ max_connections = 1000
 
 # The following are important to configure and depend a lot on to the size of
 # your database and the available resources.
-#innodb_buffer_pool_size = 4G
+innodb_buffer_pool_size = 8G
 #innodb_log_file_size = 256M
 #innodb_log_buffer_size = 8M
 

stack/erp5/apache.conf.in

@@ -68,6 +68,6 @@ Listen {{ ip }}:{{ port }}
 {%   endfor -%}
 <VirtualHost *:{{ port }}>
   SSLEngine on
-  RewriteRule ^/(.*) {{ backend }}/$1 [L,P]
+  RewriteRule ^(/_user|/_anon)?/(.*) {{ backend }}/$2 [L,P]
 </VirtualHost>
 {% endfor -%}

stack/erp5/buildout.cfg

@@ -67,6 +67,8 @@ extends =
   ../../component/6tunnel/buildout.cfg
   ../../component/findutils/buildout.cfg
   ../../component/userhosts/buildout.cfg
+  ../../stack/monitor/buildout.cfg
+  ../../component/pycurl/buildout.cfg
 
 parts =
   rdiff-backup
@@ -176,10 +178,29 @@ context =
     key slapos_core_version versions:slapos.core
     ${:extra-context}
 
+[template-http-monitor]
+recipe = slapos.recipe.template
+filename = instance-http-monitor.cfg.in
+url = ${:_profile_base_location_}/${:filename}
+md5sum = 7fb20468f1825a82951a222df9c8b045
+output = ${buildout:directory}/template-http-monitor.cfg.in
+
+##################e
+# Monitor element
+#
+
+[template-monitor-check-cache-hit]
+< = download-base
+url = ${:_profile_base_location_}/monitor-templates/monitor-check-cache-hit.in
+download-only = true
+md5sum = 8e9ea2bec2fff4bd98c074cb69cf8592
+filename = monitor-check-cache-hit.in
+mode = 0644
+
 [template-mariadb]
 < = download-base
 filename = instance-mariadb.cfg.in
-md5sum = 85826f72b4eac9cbaea2d34d73e6d445
+md5sum = 9bf570924391ceb5fc0332a614955c2a
 link-binary =
   ${coreutils:location}/bin/basename
   ${coreutils:location}/bin/cat
@@ -237,7 +258,7 @@ md5sum = e40e2e39f4941c6372f4357e8589a5cf
 # XXX: "template.cfg" is hardcoded in instanciation recipe
 filename = template.cfg
 template = ${:_profile_base_location_}/instance.cfg.in
-md5sum = 8ccd836753a44aa26d68f5fff00c2eff
+md5sum = 510df07237978939660364a70b79158c
 extra-context =
     key mariadb_link_binary template-mariadb:link-binary
     key zope_link_binary template-zope:link-binary
@@ -287,10 +308,12 @@ extra-context =
     key template_create_erp5_site_real template-create-erp5-site-real:target
     key template_erp5 template-erp5:target
     key template_haproxy_cfg template-haproxy-cfg:target
+    key template_http_monitor template-http-monitor:output
     key template_kumofs template-kumofs:target
     key template_logrotate_base template-logrotate-base:rendered
     key template_mariadb template-mariadb:target
     key template_mariadb_initial_setup template-mariadb-initial-setup:target
+    key template_monitor monitor-template:output
     key template_my_cnf template-my-cnf:target
     key template_neo template-neo:target
     key template_runzope_userhosts_preloaded template-runzope-userhosts-preloaded:target
@@ -304,7 +327,7 @@ extra-context =
 [template-erp5]
 < = download-base
 filename = instance-erp5.cfg.in
-md5sum = b628da98dbed2f7e46f0de207b4bd228
+md5sum = 23a6e5e143ac0bfeef9416d2b262c41d
 
 [template-neo]
 < = download-base
@@ -319,7 +342,7 @@ md5sum = 7bbb690cb2ea38cd2aa84c8a79c50399
 [template-zope]
 < = download-base
 filename = instance-zope.cfg.in
-md5sum = e67bf593eb980761c61f126417c67b0c
+md5sum = d551d5739a3c5b1c3aeff67e1410f240
 link-binary =
   ${aspell:location}/bin/aspell
   ${coreutils:location}/bin/basename
@@ -354,7 +377,7 @@ md5sum = 86131104949100316eece4bcae8ee1b8
 [template-apache-conf]
 < = download-base
 filename = apache.conf.in
-md5sum = dedfe759c3421e0186703fe1257bae8f
+md5sum = 399ea8b0b8bdcc63d7295351445a863a
 
 [template-haproxy-cfg]
 < = download-base
@@ -573,6 +596,13 @@ eggs =
 # Needed for parsing .po files from our Localizer subset
   polib
 
+# Needod for stack monitor
+  cns.recipe.symlink
+  collective.recipe.template
+
+# needed for monitoring tool
+  ${pycurl:egg}
+
 # parameterizing the version of the generated python interpreter name by the
 # python section version causes dependency between this egg section and the
 # installation of python, which we don't want on an instance

stack/erp5/instance-erp5.cfg.in

@@ -46,6 +46,7 @@ config-{{ option }} = {{ dumps(value) }}
 {# Fail early if an unexpected value is provided -#}
 {% set possible_software_type_dict = {'zeo': 'zeo', 'neo': 'neo'} -%}
 {{ request('zodb', 'zodb-' ~ possible_software_type_dict[slapparameter_dict.get('zodb-software-type', 'zeo')], 'zodb', {'tcpv4-port': 2100, 'zodb-dict': {'root': {}}}, {'zodb-storage-type': False, 'zodb-dict': False, 'tidstorage-ip': False, 'tidstorage-port': False}) }}
+{{ request('http-monitor', 'http-monitor', 'http-monitor', {}, {'monitor-url': True}) }}
 
 [inituser-password]
 {% set inituser_password = slapparameter_dict.get('inituser-password') -%}

stack/erp5/instance-http-monitor.cfg.in

+{% if software_type == slap_software_type -%}
+[buildout]
+parts =
+  certificate-authority
+  cron-entry-monitor
+  cron-entry-rss
+  cron
+  deploy-index
+  deploy-settings-cgi
+  deploy-status-cgi
+  deploy-status-history-cgi
+  setup-static-files
+  certificate-authority
+  zero-parameters
+  public-symlink
+  cgi-httpd-wrapper
+  cgi-httpd-graceful-wrapper
+  monitor-promise
+  monitor-instance-log-access
+  monitor-check-cache-hit
+  publish-http-monitor
+
+extends = ${monitor-template:output}
+
+eggs-directory = {{ eggs_directory }}
+develop-eggs-directory = {{ develop_eggs_directory }}
+
+[monitor-check-cache-hit]
+recipe = slapos.recipe.template:jinja2
+template = ${template-monitor-check-cache-hit:location}/${template-monitor-check-cache-hit:filename}
+rendered = $${monitor-directory:monitor-custom-scripts}/check-cache-hit.py
+mode = 700
+context =
+  raw config_cfg $${buildout:directory}/$${public:filename}
+  raw python_executable ${buildout:bin-directory}/python2.7
+
+[public]
+recipe = slapos.cookbook:zero-knowledge.write
+filename = knowledge0.cfg
+json_url_list = ["http://www.erp5.com/"]
+json_resolve_list = ["www.erp5.com:80:5.135.149.226"]
+
+[zero-parameters]
+recipe = slapos.cookbook:zero-knowledge.read
+filename = $${public:filename}
+
+[publish-http-monitor]
+recipe = slapos.cookbook:publish.serialised
+monitor-url = $${monitor-parameters:url}
+
+{% endif %}

stack/erp5/instance-mariadb.cfg.in

@@ -32,6 +32,7 @@ recipe = slapos.cookbook:publish.serialised
 {% endmacro -%}
 database-list = {{ render_database_list(database_list) }}
 test-database-list = {{ render_database_list(test_database_list) }}
+monitor-url = ${publish-connection-informations:monitor_url}
 
 [simplefile]
 recipe = slapos.recipe.template:jinja2
@@ -218,9 +219,21 @@ path = ${directory:promise}/mariadb
 hostname = ${my-cnf-parameters:ip}
 port = ${my-cnf-parameters:port}
 
+[{{ section('monitor-current-log-access') }}]
+< = monitor-directory-access
+source = ${directory:log}
+
+[{{ section('monitor-backup-log-access') }}]
+< = monitor-directory-access
+source = ${logrotate-entry-mariadb:backup}
+
+[monitor-parameters]
+port = 3380
+
 [buildout]
 extends =
   {{ parameter_dict['instance-logrotate-cfg'] }}
+  {{ parameter_dict['template-monitor'] }}
 parts +=
   publish-mariadb-url
   logrotate-entry-mariadb
@@ -228,6 +241,25 @@ parts +=
   update-mysql
   mysqld
   promise
+
+# Access to mariadb logs
+  certificate-authority
+  cron-entry-monitor
+  cron-entry-rss
+  deploy-index
+#  deploy-settings-cgi
+#  deploy-status-cgi
+#  deploy-status-history-cgi
+  setup-static-files
+  certificate-authority
+  zero-parameters
+  public-symlink
+  cgi-httpd-wrapper
+  cgi-httpd-graceful-wrapper
+  monitor-promise
+  monitor-instance-log-access
+
+# Complete parts with sections
   {{ part_list | join('\n  ') }}
 
 eggs-directory = {{ eggs_directory }}

stack/erp5/instance-zope.cfg.in

@@ -79,6 +79,7 @@ ca-private = ${:ca-dir}/private
 ca-certs = ${:ca-dir}/certs
 ca-newcerts = ${:ca-dir}/newcerts
 ca-crl = ${:ca-dir}/crl
+logrotate-backup = ${:var}/logrotate
 
 [binary-link]
 recipe = slapos.cookbook:symbolic.link
@@ -99,6 +100,25 @@ recipe = slapos.cookbook:certificate_authority
 openssl-binary = ${binary-link:target-directory}/openssl
 wrapper = ${directory:services}/ca
 
+[{{ section('monitor-current-log-access') }}]
+< = monitor-directory-access
+source = ${directory:log}
+
+[{{ section('monitor-backup-log-access') }}]
+< = monitor-directory-access
+source = ${directory:logrotate-backup}
+
+[monitor-parameters]
+port = {{ slapparameter_dict['port-base'] + 5000 }}
+
+[public]
+recipe = slapos.cookbook:zero-knowledge.write
+filename = knowledge0.cfg
+
+[zero-parameters]
+recipe = slapos.cookbook:zero-knowledge.read
+filename = $${public:filename}
+
 {% if use_ipv6 -%}
 {%   set ipv6 = (ipv6_set | list)[0] -%}
 [zeo-tunnel-base]
@@ -350,6 +370,7 @@ dict toward erp5 partition, violating the DRY principle and making the intent
 hard to guess.
 -#}
 hosts-dict = {{ dumps(hosts_dict) }}
+monitor-url = ${monitor-parameters:url}
 
 [erp5-promise]
 recipe = slapos.cookbook:erp5.promise
@@ -366,8 +387,24 @@ eggs-directory = {{ eggs_directory }}
 develop-eggs-directory = {{ develop_eggs_directory }}
 extends =
   {{ parameter_dict['instance-logrotate-cfg'] }}
+  {{ parameter_dict['template-monitor'] }}
 parts +=
   erp5-promise
+  certificate-authority
+  cron-entry-monitor
+  cron-entry-rss
+  deploy-index
+#  deploy-settings-cgi
+#  deploy-status-cgi
+#  deploy-status-history-cgi
+  setup-static-files
+  certificate-authority
+  zero-parameters
+  public-symlink
+  cgi-httpd-wrapper
+  cgi-httpd-graceful-wrapper
+  monitor-promise
+  monitor-instance-log-access
   {{ part_list | join('\n  ') }}
   publish-zope
 versions = versions

stack/erp5/instance.cfg.in

@@ -143,6 +143,7 @@ jsl = {{ jsl_location }}
 link-binary = {{ dumps(zope_link_binary) }}
 userhosts = {{ userhosts_location }}
 runzope-userhosts-preloaded-template = {{ template_runzope_userhosts_preloaded }}
+template-monitor = {{ template_monitor }}
 
 [dynamic-template-zope]
 < = jinja2-template-base
@@ -188,6 +189,7 @@ template-my-cnf = {{ template_my_cnf }}
 template-mariadb-initial-setup = {{ template_mariadb_initial_setup }}
 link-binary = {{ dumps(mariadb_link_binary) }}
 instance-logrotate-cfg = {{ template_logrotate_base }}
+template-monitor = {{ template_monitor }}
 
 [dynamic-template-mariadb]
 < = jinja2-template-base
@@ -214,6 +216,13 @@ extra-context =
 # Must match the key id in [switch-softwaretype] which uses this section.
     raw software_type create-erp5-site
 
+[dynamic-template-http-monitor]
+< = jinja2-template-base
+template = {{ template_http_monitor }}
+filename = instance-http-monitor.cfg
+extra-context =
+    raw software_type http-monitor
+
 [switch-softwaretype]
 recipe = slapos.recipe.build
 script =
@@ -253,3 +262,4 @@ balancer = ${dynamic-template-balancer:rendered}
 zodb-neo = ${dynamic-template-neo:rendered}
 zodb-zeo = ${dynamic-template-zeo:rendered}
 zope = ${dynamic-template-zope:rendered}
+http-monitor = ${dynamic-template-http-monitor:rendered}

stack/erp5/monitor-templates/monitor-check-cache-hit.in

+#!{{ python_executable }}
+# -*- coding: utf-8 -*-
+
+# dependency: python2-pycurl
+
+import sys
+import pycurl
+import ConfigParser
+import os
+import json
+
+from mimetools import Message
+from cStringIO import StringIO
+from HTMLParser import HTMLParser
+
+####################
+# Init logging tools
+def debug(*args):
+  # sys.stderr.write("DEBUG: " + "\n     : ".join(" ".join((str(arg) for arg in args)).split("\n")) + "\n")
+  pass
+def log(*args):
+  sys.stderr.write(" ".join((str(arg) for arg in args)) + "\n")
+  pass
+def info(*args):
+  sys.stderr.write("INFO : " + "\n     : ".join(" ".join((str(arg) for arg in args)).split("\n")) + "\n")
+  pass
+last_warn_log = None
+def warn(*args):
+  global last_warn_log
+  last_warn_log = "WARN : " + "\n     : ".join(" ".join((str(arg) for arg in args)).split("\n")) + "\n"
+  sys.stderr.write(last_warn_log)
+last_error_log = None
+def error(*args):
+  global last_error_log
+  last_error_log = "ERROR: " + "\n     : ".join(" ".join((str(arg) for arg in args)).split("\n")) + "\n"
+  sys.stderr.write(last_error_log)
+
+#############################
+# Checking configuration file
+config_file = "{{ config_cfg }}"
+if not os.path.exists(config_file):
+  error("Your software does *NOT* embed 0-knowledge. This interface is useless in this case")
+  exit(1)
+
+##################################
+# Getting configuration parameters
+config_parser = ConfigParser.ConfigParser()
+config_parser.read(config_file)
+
+if not config_parser.has_section('public'):
+  error("Your software does not use 0-knowledge settings.")
+  exit(1)
+
+url_list = json.loads(config_parser.get("public", "json_url_list"))
+resolve_list = json.loads(config_parser.get("public", "json_resolve_list"))
+
+########################
+# Prepare & run requests
+class MyHTMLParser(HTMLParser):
+  def __init__(self, base_url):
+    HTMLParser.__init__(self)
+    self.base_url = base_url
+  def handle_starttag(self, tag, attrs):
+    def handle_url(url):
+      if not url.startswith("http"):
+        url = self.base_url + url
+      do_request(url)
+    # handle images and js scripts
+    if tag == 'img' or tag == 'script':
+      debug(tag, attrs)
+      for attr in attrs:
+        if attr[0] == 'src':
+          handle_url(attr[1])
+    # handle css and favicon
+    if tag == 'link' and \
+       any(attr[0] == "rel" and attr[1] in ("stylesheet", "shortcut icon") for attr in attrs):
+      debug(tag, attrs)
+      for attr in attrs:
+        if attr[0] == "href":
+          handle_url(attr[1])
+
+headers = {
+  "User-Agent": "test",
+  "Accept-Encoding": "gzip, deflate",
+  "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8",
+  "Accept-Language": "en-US,en;q=0.5",
+ #   "Connection": "keep-alive"
+}
+
+parsed_url_dict = {}
+
+info("Start checking for cache hits")
+info("Parameters:\nurl_list:", url_list, "\nresolve_list:", resolve_list)
+
+def do_request(url):
+  if parsed_url_dict.get("url") is not None: return
+  parsed_url_dict[url] = True
+  log("Checking cache hit for", url)
+  c = pycurl.Curl()
+  response_headers = StringIO()
+  output = StringIO()
+  c.setopt(c.URL, url)
+  c.setopt(c.RESOLVE, resolve_list)
+  c.setopt(c.WRITEFUNCTION, output.write)
+  c.setopt(c.HEADERFUNCTION, response_headers.write)
+  c.perform() # perform a request before testing if the cache is hit
+  response_headers.truncate(0)
+  output.truncate(0)
+  c.perform()
+
+  response_headers.seek(0)
+  status = response_headers.readline().split(" ")[1]
+
+  m = Message(response_headers)
+
+  # see http://labs.omniti.com/people/mark/ats_sa/slides.html#slide-18
+  if any("[cHs" in header for header in m.getheaders('via')) or \
+     any("HIT" in header for header in m.getheaders("x-cache")):
+    debug("Cache hit found in 'Via' or 'X-Cache' headers")
+  else:
+    error("No cache hit found in 'Via' or 'X-Cache' headers\n" + response_headers.getvalue().rstrip())
+
+  if m.getheader('content-type', '').startswith('text/html'):
+    MyHTMLParser(url).feed(output.getvalue())
+
+  response_headers.close()
+  output.close()
+
+for url in url_list:
+  do_request(url)
+
+if last_error_log is not None:
+  sys.exit(1)
+info("OK");

stack/monitor/buildout.cfg

@@ -41,14 +41,14 @@ recipe = slapos.recipe.template
 url = ${:_profile_base_location_}/monitor.cfg.in
 output = ${buildout:directory}/monitor.cfg
 filename = monitor.cfg
-md5sum = 4d0c3b847c18a56068f04dd926487272
+md5sum = 64520d1d24b35afb52e0a24744ac2cf6
 mode = 0644
 
 [monitor-bin]
 recipe = hexagonit.recipe.download
 url = ${:_profile_base_location_}/${:filename}
 download-only = true
-md5sum = cb2f15850d3dc82459a0044adb4416cf
+md5sum = 9364ed5da70c630efb3c9a59e79a816b
 destination = ${buildout:directory}/parts/monitor-template-monitor-bin
 filename = monitor.py.in
 mode = 0644

stack/monitor/monitor.cfg.in

@@ -228,7 +228,9 @@ name = example.com
 # Deploy a webserver running cgi scripts for monitoring
 ###########
 [public]
+# XXX provide absolute path name
 recipe = slapos.cookbook:zero-knowledge.write
+# XXX zero knowledge does not automaticaly remove no longer used parameters from config file `filename`
 filename = knowledge0.cfg
 status-history-length = 5
 

stack/monitor/monitor.py.in

@@ -72,6 +72,22 @@ CREATE TABLE IF NOT EXISTS individual_status (
   db.commit()
   db.close()
 
+def init_db():
+  db = sqlite3.connect(db_path)
+  c = db.cursor()
+  c.executescript("""
+CREATE TABLE IF NOT EXISTS status (
+  timestamp INTEGER UNIQUE,
+  status VARCHAR(255));
+CREATE TABLE IF NOT EXISTS individual_status (
+  timestamp INTEGER,
+  status VARCHAR(255),
+  element VARCHAR(255),
+  output TEXT);
+""")
+  db.commit()
+  db.close()
+
 def getListOfScripts(directory):
   """
   Get the list of script inside of a directory (not recursive)