diff --git a/setup.py b/setup.py index a3895f5f9a9eb7cf746eb06d197083484810f234..4fc5222536dca32181c5d827b43f72a61846ef04 100644 --- a/setup.py +++ b/setup.py @@ -41,6 +41,7 @@ setup(name=name, zip_safe=True, entry_points={ 'zc.buildout': [ + 'bef_erp5 = slapos.recipe.bef_erp5:Recipe', 'build = slapos.recipe.build:Script', 'buildcmmi = slapos.recipe.build:Cmmi', 'download = slapos.recipe.download:Recipe', diff --git a/slapos/recipe/bef_erp5/__init__.py b/slapos/recipe/bef_erp5/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..9d4c503f1dad9b3c7a6374dba812f1df7507a595 --- /dev/null +++ b/slapos/recipe/bef_erp5/__init__.py @@ -0,0 +1,305 @@ +############################################################################## +# +# Copyright (c) 2010 Vifib SARL and Contributors. All Rights Reserved. +# +# WARNING: This program as such is intended to be used by professional +# programmers who take the whole responsibility of assessing all potential +# consequences resulting from its eventual inadequacies and bugs +# End users who are looking for a ready-to-use solution with commercial +# guarantees and support are strongly adviced to contract a Free Software +# Service Company +# +# This program is Free Software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License +# as published by the Free Software Foundation; either version 3 +# of the License, or (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. +# +############################################################################## +import slapos.recipe.erp5 +import os +import pkg_resources +import zc.buildout +import sys +import netaddr + +BEF_TIMEZONE = 'BST' + +def validLoopBackAddress(ip): + if netaddr.IPAddress(ip).is_loopback(): + return True + else: + return False + +def validPublicAddress(ip): + return not validLoopBackAddress(ip) + +class Recipe(slapos.recipe.erp5.Recipe): + site_id = 'erp5' + + def getLocalIPv4Address(self): + """Returns local IPv4 address available on partition""" + # XXX: Lack checking for locality of address + if self.development: + # XXX: Development superhack. + return slapos.recipe.erp5.Recipe.getLocalIPv4Address(self) + return self._getIpAddress(validLoopBackAddress) + + def getGlobalIPv6Address(self): + """Returns global IPv6 address available on partition""" + if self.development: + # XXX: Development superhack. + return slapos.recipe.erp5.Recipe.getGlobalIPv6Address(self) + # XXX: Lack checking for globality of address + + return self._getIpAddress(validPublicAddress) + + def _getZeoClusterDict(self): + site_path = '/erp5/' + return { + '/': (self._requestZeoFileStorage('Zeo Server 1', 'main'), + site_path + 'account_module'), + site_path + 'portal_activities': (self._requestZeoFileStorage('Zeo Server 1', + 'portal_activities'), None), + site_path + 'task_report_module': (self._requestZeoFileStorage('Zeo Server 1', + 'task_report_module'), None), + site_path + 'video_request_module': (self._requestZeoFileStorage('Zeo Server 1', + 'video_request_module'), None), + site_path + 'bug_module': (self._requestZeoFileStorage('Zeo Server 1', + 'bug_module'), None), + site_path + 'portal_simulation': (self._requestZeoFileStorage('Zeo Server 1', + 'portal_simulation'), None), + site_path + 'notice_transfer_request_module': (self._requestZeoFileStorage( + 'Zeo Server 1', 'notice_transfer_request_module'), None), + site_path + 'accounting_module': (self._requestZeoFileStorage('Zeo Server 1', + 'accounting_module'), None), + site_path + 'task_module': (self._requestZeoFileStorage('Zeo Server 1', + 'task_module'), None), + + site_path + 'video_module': (self._requestZeoFileStorage('Zeo Server 2', + 'video_module'), None), + + site_path + 'event_module': (self._requestZeoFileStorage('Zeo Server 3', + 'event_module'), None), + site_path + 'support_request_module': (self._requestZeoFileStorage( + 'Zeo Server 3', 'support_request_module'), None), + + site_path + 'person_module': (self._requestZeoFileStorage('Zeo Server 4', + 'person_module'), None), + site_path + 'organisation_module': (self._requestZeoFileStorage('Zeo Server 4', + 'organisation_module'), None), + site_path + 'portal_synchronizations': (self._requestZeoFileStorage( + 'Zeo Server 4', 'portal_synchronizations'), None), + + site_path + 'scanned_document_module': (self._requestZeoFileStorage( + 'Zeo Server 5', 'scanned_document_module'), None), + site_path + 'item_module': (self._requestZeoFileStorage('Zeo Server 5', + 'item_module'), None), + site_path + 'document_module': (self._requestZeoFileStorage('Zeo Server 5', + 'document_module'), None), + site_path + 'image_module': (self._requestZeoFileStorage('Zeo Server 5', + 'image_module'), None), + site_path + 'external_source_module': (self._requestZeoFileStorage( + 'Zeo Server 5', 'external_source_module'), None), + } + + def installProductionMysql(self): + mysql_conf = self.installMysqlServer(self.getLocalIPv4Address(), 45678, + template_filename=pkg_resources.resource_filename(__name__, + 'template/my.cnf.in'), parallel_test_database_amount=0, + mysql_conf=dict(innodb_buffer_pool_size='10G')) + self.setConnectionDict(dict( + mysql_url='%(mysql_database)s@%(ip)s:%(tcp_port)s %(mysql_user)s %(mysql_password)s' % mysql_conf, + )) + return self.path_list + + def installProductionApplication(self): + site_check_path = '/%s/getId' % self.site_id + ca_conf = self.installCertificateAuthority() + backend_key, backend_certificate = self.requestCertificate('Login Based Access') + memcached_conf = self.installMemcached(ip=self.getLocalIPv4Address(), + port=11000) + conversion_server_conf = self.installConversionServer( + self.getLocalIPv4Address(), 23000, 23060) + user, password = self.installERP5() + ip = self.getLocalIPv4Address() + mount_point_zeo_dict = self._getZeoClusterDict() + zeo_conf = self.installZeo(ip) + zodb_configuration_list = [] + known_tid_storage_identifier_dict = {} + for mount_point, (storage_dict, check_path) in mount_point_zeo_dict.iteritems(): + known_tid_storage_identifier_dict[ + (((storage_dict['ip'],storage_dict['port']),), storage_dict['storage_name']) + ] = (zeo_conf[storage_dict['storage_name']]['path'], check_path or mount_point) + zodb_configuration_list.append(self.substituteTemplate( + self.getTemplateFilename('zope-zeo-snippet.conf.in'), dict( + storage_name=storage_dict['storage_name'], + address='%s:%s' % (storage_dict['ip'], storage_dict['port']), + mount_point=mount_point + ))) + tidstorage_config = dict(host=self.getLocalIPv4Address(), port='6001') + zodb_configuration_string = '\n'.join(zodb_configuration_list) + zope_port = 12000 + # One Distribution Node + zope_port +=1 + self.installZope(ip, zope_port, 'zope_distribution', + with_timerservice=True, + zodb_configuration_string=zodb_configuration_string, + tidstorage_config=tidstorage_config, + zope_environment=dict(TZ=BEF_TIMEZONE)) + # Two Activity Nodes + for i in (1, 2): + zope_port += 1 + self.installZope(ip, zope_port, 'zope_activity_%s' % i, + with_timerservice=True, + zodb_configuration_string=zodb_configuration_string, + tidstorage_config=tidstorage_config, + zope_environment=dict(TZ=BEF_TIMEZONE)) + # Eight Working Nodes (Human access) + login_url_list = [] + for i in (1, 2, 3, 4, 5, 6, 7, 8): + zope_port += 1 + login_url_list.append(self.installZope(ip, zope_port, + 'zope_login_%s' % i, with_timerservice=False, + zodb_configuration_string=zodb_configuration_string, + tidstorage_config=tidstorage_config, + zope_environment=dict(TZ=BEF_TIMEZONE))) + login_haproxy = self.installHaproxy(ip, 15001, 'login', site_check_path, + login_url_list) + # One Web Node + zope_port += 1 + web_url_list = [self.installZope(ip, zope_port, 'zope_web', + with_timerservice=True, + zodb_configuration_string=zodb_configuration_string, + tidstorage_config=tidstorage_config, + thread_amount=10, zope_environment=dict(TZ=BEF_TIMEZONE))] + web_haproxy = self.installHaproxy(ip, 15002, 'web', site_check_path, + web_url_list) + apache_web = self.installBackendApache(self.getGlobalIPv6Address(), 15001, + web_haproxy, backend_key, backend_certificate, suffix='_web') + + # One Admin Node + zope_port += 1 + admin_url_list = [self.installZope(ip, zope_port, 'zope_admin', + with_timerservice=True, + zodb_configuration_string=zodb_configuration_string, + tidstorage_config=tidstorage_config, + zope_environment=dict(TZ=BEF_TIMEZONE))] + admin_haproxy = self.installHaproxy(ip, 15003, 'admin', site_check_path, + admin_url_list) + apache_admin = self.installBackendApache(self.getGlobalIPv6Address(), 15002, + admin_haproxy, backend_key, backend_certificate, suffix='_admin') + + self.installTidStorage(tidstorage_config['host'], tidstorage_config['port'], + known_tid_storage_identifier_dict, 'http://'+login_haproxy) + apache_login = self.installBackendApache(self.getGlobalIPv6Address(), 15000, + login_haproxy, backend_key, backend_certificate, + access_control_string='%s' % self.getGlobalIPv6Address()) + frontend_key, frontend_certificate = self.requestCertificate( + self.parameter_dict['frontend_name']) + login_frontend = self.installFrontendZopeApache( + self.getGlobalIPv6Address(), 18000, + self.parameter_dict['frontend_name'], + self.parameter_dict['frontend_path'], + apache_login, + self.parameter_dict['backend_path'], frontend_key, + frontend_certificate, + access_control_string=self.parameter_dict['frontend_acl_string']) + memcached_conf = self.installMemcached(ip=self.getLocalIPv4Address(), + port=11000) + kumo_conf = self.installKumo(self.getLocalIPv4Address()) + self.setConnectionDict(dict( + site_url='https://%s:%s%s' % (self.getGlobalIPv6Address(), 18000, + self.parameter_dict['frontend_path']), + site_web_url=apache_web, + site_admin_url=apache_admin, + site_user=user, + site_password=password, + memcached_url=memcached_conf['memcached_url'], + kumo_url=kumo_conf['kumo_address'], + conversion_server_url='%(conversion_server_ip)s:%(conversion_server_port)s' % + conversion_server_conf, + # openssl binary might be removed, as soon as CP environment will be + # fully controlled + openssl_binary=self.options['openssl_binary'], + # As soon as there would be Vifib ERP5 configuration and possibility to + # call it over the network this can be removed + certificate_authority_path=ca_conf['certificate_authority_path'], + )) + return self.path_list + + def installDevelopmentEnvironment(self): + ca_conf = self.installCertificateAuthority() + memcached_conf = self.installMemcached(ip=self.getLocalIPv4Address(), + port=11000) + conversion_server_conf = self.installConversionServer( + self.getLocalIPv4Address(), 23000, 23060) + mysql_conf = self.installMysqlServer(self.getLocalIPv4Address(), 45678, + template_filename=pkg_resources.resource_filename(__name__, + 'template/my.cnf.in'), parallel_test_database_amount=0, + mysql_conf=dict(innodb_buffer_pool_size='1G')) + kumo_conf = self.installKumo(self.getLocalIPv4Address()) + user, password = self.installERP5() + self.installTestRunner(ca_conf, mysql_conf, conversion_server_conf, + memcached_conf, kumo_conf) + ip = self.getLocalIPv4Address() + zope_port = '18080' + zodb_dir = os.path.join(self.data_root_directory, 'zodb') + self._createDirectory(zodb_dir) + zodb_root_path = os.path.join(zodb_dir, 'root.fs') + self.installZope(ip, zope_port, 'zope_development', + zodb_configuration_string=self.substituteTemplate( + self.getTemplateFilename('zope-zodb-snippet.conf.in'), + dict(zodb_root_path=zodb_root_path)), + thread_amount=8, with_timerservice=True, + zope_environment=dict(TZ=BEF_TIMEZONE)) + self.setConnectionDict(dict( + site_user=user, + site_password=password, + memcached_url=memcached_conf['memcached_url'], + kumo_url=kumo_conf['kumo_address'], + conversion_server_url='%(conversion_server_ip)s:%(conversion_server_port)s' % + conversion_server_conf, + # openssl binary might be removed, as soon as CP environment will be + # fully controlled + openssl_binary=self.options['openssl_binary'], + # As soon as there would be Vifib ERP5 configuration and possibility to + # call it over the network this can be removed + certificate_authority_path=ca_conf['certificate_authority_path'], + # as installERP5Site is not trusted (yet) and this recipe is production + # ready expose more information + mysql_url='%(mysql_database)s@%(ip)s:%(tcp_port)s %(mysql_user)s %(mysql_password)s' % mysql_conf, + development_zope='http://%s:%s/' % (ip, zope_port) + )) + return self.path_list + + def _install(self): + self.path_list = [] + self.requirements, self.ws = self.egg.working_set([__name__]) + # self.cron_d is a directory, where cron jobs can be registered + self.cron_d = self.installCrond() + self.logrotate_d, self.logrotate_backup = self.installLogrotate() + self.killpidfromfile = zc.buildout.easy_install.scripts( + [('killpidfromfile', 'slapos.recipe.erp5.killpidfromfile', + 'killpidfromfile')], self.ws, sys.executable, self.bin_directory)[0] + self.path_list.append(self.killpidfromfile) + self.linkBinary() + if self.parameter_dict.get('development', 'false').lower() == 'true': + self.development = True + return self.installDevelopmentEnvironment() + if self.parameter_dict.get('production_mysql', 'false').lower() == 'true': + self.development = False + return self.installProductionMysql() + if self.parameter_dict.get('production_application', 'false').lower() == 'true': + self.development = False + return self.installProductionApplication() + + raise NotImplementedError('Flavour of instance have to be given.') diff --git a/slapos/recipe/bef_erp5/template/my.cnf.in b/slapos/recipe/bef_erp5/template/my.cnf.in new file mode 100644 index 0000000000000000000000000000000000000000..646257c526587886d6bfcc09018d9fca8865ea65 --- /dev/null +++ b/slapos/recipe/bef_erp5/template/my.cnf.in @@ -0,0 +1,186 @@ +# Example MySQL config file for medium systems. +# +# This is for a system with little memory (32M - 64M) where MySQL plays +# an important part, or systems up to 128M where MySQL is used together with +# other programs (such as a web server) +# + +# The following options will be passed to all MySQL clients +[client] +user = root +#password = your_password +port = %(tcp_port)s +socket = %(socket)s + +# Here follows entries for some specific programs + +# The MySQL server +[mysqld] +#user = mysql # not needed in SlapOS +datadir = %(data_directory)s +port = %(tcp_port)s +socket = %(socket)s +pid-file = %(pid_file)s +skip-locking +key_buffer = 128M +max_allowed_packet = 100M +table_cache = 1024 +sort_buffer_size = 100M +net_buffer_length = 8K +read_buffer_size = 256K +read_rnd_buffer_size = 512K +myisam_sort_buffer_size = 8M +collation_server = utf8_unicode_ci +character_set_server = utf8 + +log-slow-queries=%(slow_query_log)s +long_query_time=1 + +bind-address = %(ip)s # new in SlapOS +log-error = %(error_log)s # new in SlapOS + +# max_connections = 256 +max_connections = 512 +# Default to using old password format for compatibility with old and +# shorter password hash. +# Reference: http://dev.mysql.com/doc/mysql/en/Password_hashing.html +old_passwords + +# Don't listen on a TCP/IP port at all. This can be a security enhancement, +# if all processes that need to connect to mysqld run on the same host. +# All interaction with mysqld must be made via Unix sockets or named pipes. +# Note that using this option without enabling named pipes on Windows +# (via the "enable-named-pipe" option) will render mysqld useless! +# +# skip-networking + +# Replication Master Server (default) +# binary logging is required for replication +#log-bin=mysql-bin + +# required unique id between 1 and 2^32 - 1 +# defaults to 1 if master-host is not set +# but will not function as a master if omitted +server-id = 1 + +# Replication Slave (comment out master section to use this) +# +# To configure this host as a replication slave, you can choose between +# two methods : +# +# 1) Use the CHANGE MASTER TO command (fully described in our manual) - +# the syntax is: +# +# CHANGE MASTER TO MASTER_HOST=<host>, MASTER_PORT=<port>, +# MASTER_USER=<user>, MASTER_PASSWORD=<password> ; +# +# where you replace <host>, <user>, <password> by quoted strings and +# <port> by the master's port number (%(tcp_port)s by default). +# +# Example: +# +# CHANGE MASTER TO MASTER_HOST='125.564.12.1', MASTER_PORT=%(tcp_port)s, +# MASTER_USER='joe', MASTER_PASSWORD='secret'; +# +# OR +# +# 2) Set the variables below. However, in case you choose this method, then +# start replication for the first time (even unsuccessfully, for example +# if you mistyped the password in master-password and the slave fails to +# connect), the slave will create a master.info file, and any later +# change in this file to the variables' values below will be ignored and +# overridden by the content of the master.info file, unless you shutdown +# the slave server, delete master.info and restart the slaver server. +# For that reason, you may want to leave the lines below untouched +# (commented) and instead use CHANGE MASTER TO (see above) +# +# required unique id between 2 and 2^32 - 1 +# (and different from the master) +# defaults to 2 if master-host is set +# but will not function as a slave if omitted +#server-id = 2 +# +# The replication master for this slave - required +#master-host = <hostname> +# +# The username the slave will use for authentication when connecting +# to the master - required +#master-user = <username> +# +# The password the slave will authenticate with when connecting to +# the master - required +#master-password = <password> +# +# The port the master is listening on. +# optional - defaults to %(tcp_port)s +#master-port = <port> +# +# binary logging - not required for slaves, but recommended +#log-bin=mysql-bin + +# Point the following paths to different dedicated disks +#tmpdir = /tmp/ +#log-update = /path-to-dedicated-directory/hostname + +# Uncomment the following if you are using BDB tables +#bdb_cache_size = 4M +#bdb_max_lock = 10000 + +# Uncomment the following if you are using InnoDB tables +#innodb_data_home_dir = /var/lib/mysql/ +#innodb_data_file_path = ibdata1:10M:autoextend +#innodb_log_group_home_dir = /var/lib/mysql/ +#innodb_log_arch_dir = /var/lib/mysql/ +# You can set .._buffer_pool_size up to 50 - 80 %% +# of RAM but beware of setting memory usage too high +innodb_buffer_pool_size = %(innodb_buffer_pool_size)s +innodb_additional_mem_pool_size = 40M +# Set .._log_file_size to 25 %% of buffer pool size +#innodb_log_file_size = 5M +#innodb_log_buffer_size = 8M +#innodb_flush_log_at_trx_commit = 1 +#innodb_lock_wait_timeout = 50 + +innodb_locks_unsafe_for_binlog=1 + +#bind-address=192.168.100.1 + +## Options for mysqld process: +#ndbcluster # run NDB engine +#ndb-connectstring=192.168.0.10 # location of MGM node + +## Options for ndbd process: +#[mysql_cluster] +#ndb-connectstring=192.168.0.10 # location of MGM node +# + +# Use utf8 as the default character set. +default-character-set=utf8 +# Ignore client information and use the default server character set. +skip-character-set-client-handshake + + +[mysqldump] +quick +max_allowed_packet = 16M + +[mysql] +no-auto-rehash +# Remove the next comment character if you are not familiar with SQL +#safe-updates +default-character-set = utf8 + +[isamchk] +key_buffer = 20M +sort_buffer_size = 20M +read_buffer = 2M +write_buffer = 2M + +[myisamchk] +key_buffer = 20M +sort_buffer_size = 20M +read_buffer = 2M +write_buffer = 2M + +[mysqlhotcopy] +interactive-timeout