Skip to content

S
slapos-mynij-dev
Commit 4a8671fb authored by Łukasz Nowak's avatar Łukasz Nowak
Browse files
Options

Drop already implemented parts.

parent 7c95b601
Hide whitespace changes
Inline Side-by-side
Showing with 0 additions and 505 deletions
slapos/recipe/erp5/__init__.py
View file @ 4a8671fb
......@@ -89,13 +89,6 @@ class Recipe(BaseSlapRecipe):
backend_url=apache_conf['apache_login'], key=frontend_key,
certificate=frontend_certificate)
# default_bt5_list = []
# if self.parameter_dict.get("flavour", "default") == 'configurator':
# default_bt5_list = self.options.get("configurator_bt5_list", '').split()
#
# self.installERP5Site(user, password, zope_access, mysql_conf,
# conversion_server_conf, memcached_conf, kumo_conf,
# self.site_id, default_bt5_list, ca_conf)
self.installTestRunner(ca_conf, mysql_conf, conversion_server_conf,
memcached_conf, kumo_conf)
......@@ -116,235 +109,6 @@ class Recipe(BaseSlapRecipe):
self.setConnectionDict(connection_dict)
return self.path_list
def installKeyAuthorisationApache(self, ipv6, port, backend, key, certificate,
ca_conf, key_auth_path='/'):
if ipv6:
ip = self.getGlobalIPv6Address()
else:
ip = self.getLocalIPv4Address()
ssl_template = """SSLEngine on
SSLVerifyClient require
RequestHeader set REMOTE_USER %%{SSL_CLIENT_S_DN_CN}s
SSLCertificateFile %(key_auth_certificate)s
SSLCertificateKeyFile %(key_auth_key)s
SSLCACertificateFile %(ca_certificate)s
SSLCARevocationPath %(ca_crl)s"""
apache_conf = self._getApacheConfigurationDict('key_auth_apache', ip, port)
apache_conf['ssl_snippet'] = ssl_template % dict(
key_auth_certificate=certificate,
key_auth_key=key,
ca_certificate=ca_conf['ca_certificate'],
ca_crl=ca_conf['ca_crl']
)
prefix = 'ssl_key_auth_apache'
rewrite_rule_template = \
"RewriteRule (.*) http://%(backend)s%(key_auth_path)s$1 [L,P]"
path_template = pkg_resources.resource_string('slapos.recipe.erp5',
'template/apache.zope.conf.path.in')
path = path_template % dict(path='/')
d = dict(
path=path,
backend=backend,
backend_path='/',
port=apache_conf['port'],
vhname=path.replace('/', ''),
key_auth_path=key_auth_path,
)
rewrite_rule = rewrite_rule_template % d
apache_conf.update(**dict(
path_enable=path,
rewrite_rule=rewrite_rule
))
apache_config_file = self.createConfigurationFile(prefix + '.conf',
pkg_resources.resource_string('slapos.recipe.erp5',
'template/apache.zope.conf.in') % apache_conf)
self.path_list.append(apache_config_file)
self.path_list.extend(zc.buildout.easy_install.scripts([(
'key_auth_apache',
'slapos.recipe.erp5.apache', 'runApache')], self.ws,
sys.executable, self.wrapper_directory, arguments=[
dict(
required_path_list=[certificate, key, ca_conf['ca_certificate'],
ca_conf['ca_crl']],
binary=self.options['httpd_binary'],
config=apache_config_file
)
]))
if ipv6:
return 'https://[%(ip)s:%(port)s]' % apache_conf
else:
return 'https://%(ip)s:%(port)s' % apache_conf
def installZopeCluster(self, ca_conf=None):
""" Install ERP5 using ZEO Cluster
"""
site_check_path = '/%s/getId' % self.site_id
thread_amount_per_zope = int(self.options.get(
'cluster_zope_thread_amount', 1))
activity_node_amount = int(self.options.get(
"cluster_activity_node_amount", 2))
user_node_amount = int(self.options.get(
"cluster_user_node_amount", 2))
key_auth_node_amount = int(self.options.get(
"key_auth_node_amount", 0))
ip = self.getLocalIPv4Address()
# storage_dict = self._requestZeoFileStorage('Zeo Server 1', 'main')
# zeo_conf = self.installZeo(ip)
tidstorage_config = dict(host=ip, port='6001')
# XXX How to define good values for this?
mount_point = '/'
zodb_cache_size = 5000
zeo_client_cache_size = '20MB'
check_path = '/erp5/account_module'
known_tid_storage_identifier_dict = {}
known_tid_storage_identifier_dict[
(((storage_dict['ip'],storage_dict['port']),), storage_dict['storage_name'])
] = (zeo_conf[storage_dict['storage_name']]['path'], check_path or mount_point)
zodb_configuration_string = self.substituteTemplate(
self.getTemplateFilename('zope-zeo-snippet.conf.in'), dict(
storage_name=storage_dict['storage_name'],
address='%s:%s' % (storage_dict['ip'], storage_dict['port']),
mount_point=mount_point, zodb_cache_size=zodb_cache_size,
zeo_client_cache_size=zeo_client_cache_size
))
zope_port = 12000
# One Distribution Node (Single Thread Always)
zope_port += 1
self.installZope(ip, zope_port, 'zope_distribution', with_timerservice=True,
zodb_configuration_string=zodb_configuration_string,
tidstorage_config=tidstorage_config)
# Activity Nodes (Single Thread Always)
for i in range(activity_node_amount):
zope_port += 1
self.installZope(ip, zope_port, 'zope_activity_%s' % i,
with_timerservice=True,
zodb_configuration_string=zodb_configuration_string,
tidstorage_config=tidstorage_config)
# Four Web Page Nodes (Human access)
login_url_list = []
for i in range(user_node_amount):
zope_port += 1
login_url_list.append(self.installZope(ip, zope_port,
'zope_login_%s' % i, with_timerservice=False,
zodb_configuration_string=zodb_configuration_string,
tidstorage_config=tidstorage_config,
thread_amount=thread_amount_per_zope))
login_haproxy = self.installHaproxy(ip, 15001, 'login',
site_check_path, login_url_list)
key_access = None
if key_auth_node_amount > 0:
service_url_list = []
for i in range(key_auth_node_amount):
zope_port += 1
service_url_list.append(self.installZope(ip, zope_port,
'zope_service_%s' % i, with_timerservice=False,
zodb_configuration_string=zodb_configuration_string,
tidstorage_config=tidstorage_config))
service_haproxy = self.installHaproxy(ip, 15000, 'service',
site_check_path, service_url_list)
key_auth_key, key_auth_certificate = self.requestCertificate(
'Key Based Access')
key_access = self.installKeyAuthorisationApache(True, 15500,
service_haproxy, key_auth_key, key_auth_certificate, ca_conf)
self.installTidStorage(tidstorage_config['host'],
tidstorage_config['port'],
known_tid_storage_identifier_dict, 'http://' + login_haproxy)
return login_url_list[-1], login_haproxy, key_access
def _requestZeoFileStorage(self, server_name, storage_name):
"""Local, slap.request compatible, call to ask for filestorage on Zeo
filter_kw can be used to select specific Zeo server
Someday in future it will be possible to invoke:
self.request(
software_release=self.computer_partition.getSoftwareRelease().getURI(),
software_type='Zeo Server',
partition_reference=storage_name,
filter_kw={'server_name': server_name},
shared=True
)
Thanks to this it will be possible to select precisely on which server
which storage will be placed.
"""
base_port = 35001
if getattr(self, '_zeo_storage_dict', None) is None:
self._zeo_storage_dict = {}
if getattr(self, '_zeo_storage_port_dict', None) is None:
self._zeo_storage_port_dict = {}
self._zeo_storage_port_dict.setdefault(server_name,
base_port+len(self._zeo_storage_port_dict))
self._zeo_storage_dict[server_name] = self._zeo_storage_dict.get(
server_name, []) + [storage_name]
return dict(
ip=self.getLocalIPv4Address(),
port=self._zeo_storage_port_dict[server_name],
storage_name=storage_name
)
def installLogrotate(self):
"""Installs logortate main configuration file and registers its to cron"""
logrotate_d = os.path.abspath(os.path.join(self.etc_directory,
'logrotate.d'))
self._createDirectory(logrotate_d)
logrotate_backup = self.createBackupDirectory('logrotate')
logrotate_conf = self.createConfigurationFile("logrotate.conf",
"include %s" % logrotate_d)
logrotate_cron = os.path.join(self.cron_d, 'logrotate')
state_file = os.path.join(self.data_root_directory, 'logrotate.status')
open(logrotate_cron, 'w').write('0 0 * * * %s -s %s %s' %
(self.options['logrotate_binary'], state_file, logrotate_conf))
self.path_list.extend([logrotate_d, logrotate_conf, logrotate_cron])
return logrotate_d, logrotate_backup
def registerLogRotation(self, name, log_file_list, postrotate_script):
"""Register new log rotation requirement"""
open(os.path.join(self.logrotate_d, name), 'w').write(
self.substituteTemplate(self.getTemplateFilename(
'logrotate_entry.in'),
dict(file_list=' '.join(['"'+q+'"' for q in log_file_list]),
postrotate=postrotate_script, olddir=self.logrotate_backup)))
def linkBinary(self):
"""Links binaries to instance's bin directory for easier exposal"""
for linkline in self.options.get('link_binary_list', '').splitlines():
if not linkline:
continue
target = linkline.split()
if len(target) == 1:
target = target[0]
path, linkname = os.path.split(target)
else:
linkname = target[1]
target = target[0]
link = os.path.join(self.bin_directory, linkname)
if os.path.lexists(link):
if not os.path.islink(link):
raise zc.buildout.UserError(
'Target link already %r exists but it is not link' % link)
os.unlink(link)
os.symlink(target, link)
self.logger.debug('Created link %r -> %r' % (link, target))
self.path_list.append(link)
def installSphinxSearchd(self, ip, port=9312, sql_port=9306):
data_directory = self.createDataDirectory('sphinx')
sphinx_conf_path = self.createConfigurationFile('sphinx.conf',
......@@ -443,275 +207,6 @@ SSLCARevocationPath %(ca_crl)s"""
)])[0]
self.path_list.append(command)
def installCrond(self):
timestamps = self.createDataDirectory('cronstamps')
cron_output = os.path.join(self.log_directory, 'cron-output')
self._createDirectory(cron_output)
catcher = zc.buildout.easy_install.scripts([('catchcron',
__name__ + '.catdatefile', 'catdatefile')], self.ws, sys.executable,
self.bin_directory, arguments=[cron_output])[0]
self.path_list.append(catcher)
cron_d = os.path.join(self.etc_directory, 'cron.d')
crontabs = os.path.join(self.etc_directory, 'crontabs')
self._createDirectory(cron_d)
self._createDirectory(crontabs)
wrapper = zc.buildout.easy_install.scripts([('crond',
'slapos.recipe.librecipe.execute', 'execute')], self.ws, sys.executable,
self.wrapper_directory, arguments=[
self.options['dcrond_binary'].strip(), '-s', cron_d, '-c', crontabs,
'-t', timestamps, '-f', '-l', '5', '-M', catcher]
)[0]
self.path_list.append(wrapper)
return cron_d
def requestCertificate(self, name):
hash = hashlib.sha512(name).hexdigest()
key = os.path.join(self.ca_private, hash + self.ca_key_ext)
certificate = os.path.join(self.ca_certs, hash + self.ca_crt_ext)
parser = ConfigParser.RawConfigParser()
parser.add_section('certificate')
parser.set('certificate', 'name', name)
parser.set('certificate', 'key_file', key)
parser.set('certificate', 'certificate_file', certificate)
parser.write(open(os.path.join(self.ca_request_dir, hash), 'w'))
return key, certificate
def installCertificateAuthority(self, ca_country_code='XX',
ca_email='xx@example.com', ca_state='State', ca_city='City',
ca_company='Company'):
backup_path = self.createBackupDirectory('ca')
self.ca_dir = os.path.join(self.data_root_directory, 'ca')
self._createDirectory(self.ca_dir)
self.ca_request_dir = os.path.join(self.ca_dir, 'requests')
self._createDirectory(self.ca_request_dir)
config = dict(ca_dir=self.ca_dir, request_dir=self.ca_request_dir)
self.ca_private = os.path.join(self.ca_dir, 'private')
self.ca_certs = os.path.join(self.ca_dir, 'certs')
self.ca_crl = os.path.join(self.ca_dir, 'crl')
self.ca_newcerts = os.path.join(self.ca_dir, 'newcerts')
self.ca_key_ext = '.key'
self.ca_crt_ext = '.crt'
for d in [self.ca_private, self.ca_crl, self.ca_newcerts, self.ca_certs]:
self._createDirectory(d)
for f in ['crlnumber', 'serial']:
if not os.path.exists(os.path.join(self.ca_dir, f)):
open(os.path.join(self.ca_dir, f), 'w').write('01')
if not os.path.exists(os.path.join(self.ca_dir, 'index.txt')):
open(os.path.join(self.ca_dir, 'index.txt'), 'w').write('')
openssl_configuration = os.path.join(self.ca_dir, 'openssl.cnf')
config.update(
working_directory=self.ca_dir,
country_code=ca_country_code,
state=ca_state,
city=ca_city,
company=ca_company,
email_address=ca_email,
)
self._writeFile(openssl_configuration, pkg_resources.resource_string(
__name__, 'template/openssl.cnf.ca.in') % config)
self.path_list.extend(zc.buildout.easy_install.scripts([
('certificate_authority',
__name__ + '.certificate_authority', 'runCertificateAuthority')],
self.ws, sys.executable, self.wrapper_directory, arguments=[dict(
openssl_configuration=openssl_configuration,
openssl_binary=self.options['openssl_binary'],
certificate=os.path.join(self.ca_dir, 'cacert.pem'),
key=os.path.join(self.ca_private, 'cakey.pem'),
crl=os.path.join(self.ca_crl),
request_dir=self.ca_request_dir
)]))
# configure backup
backup_cron = os.path.join(self.cron_d, 'ca_rdiff_backup')
open(backup_cron, 'w').write(
'''0 0 * * * %(rdiff_backup)s %(source)s %(destination)s'''%dict(
rdiff_backup=self.options['rdiff_backup_binary'],
source=self.ca_dir,
destination=backup_path))
self.path_list.append(backup_cron)
return dict(
ca_certificate=os.path.join(config['ca_dir'], 'cacert.pem'),
ca_crl=os.path.join(config['ca_dir'], 'crl'),
certificate_authority_path=config['ca_dir']
)
def installZeo(self, ip):
# zodb_dir = os.path.join(self.data_root_directory, 'zodb')
# self._createDirectory(zodb_dir)
zeo_configuration_dict = {}
zeo_number = 0
for zeo_server in sorted(self._zeo_storage_dict.iterkeys()):
zeo_number += 1
# zeo_event_log = os.path.join(self.log_directory, 'zeo-%s.log'% zeo_number)
# zeo_pid = os.path.join(self.run_directory, 'zeo-%s.pid'% zeo_number)
# self.registerLogRotation('zeo-%s' % zeo_number, [zeo_event_log],
# self.killpidfromfile + ' ' + zeo_pid + ' SIGUSR2')
# config = dict(
# zeo_ip=ip,
# zeo_port=self._zeo_storage_port_dict[zeo_server],
# zeo_event_log=zeo_event_log,
# zeo_pid=zeo_pid,
# )
# storage_definition_list = []
# for storage_name in sorted(self._zeo_storage_dict[zeo_server]):
# path = os.path.join(zodb_dir, '%s.fs' % storage_name)
# storage_definition_list.append("""<filestorage %(storage_name)s>
# path %(path)s
# </filestorage>"""% dict(storage_name=storage_name, path=path))
# zeo_configuration_dict[storage_name] = dict(
# ip=ip,
# port=config['zeo_port'],
# path=path
# )
# config['zeo_filestorage_snippet'] = '\n'.join(storage_definition_list)
# zeo_conf_path = self.createConfigurationFile('zeo-%s.conf' % zeo_number,
# self.substituteTemplate(self.getTemplateFilename('zeo.conf.in'), config))
# self.path_list.append(zeo_conf_path)
# wrapper = zc.buildout.easy_install.scripts([('zeo_%s' % zeo_number,
# 'slapos.recipe.librecipe.execute', 'execute')], self.ws, sys.executable,
# self.wrapper_directory, arguments=[
# self.options['runzeo_binary'].strip(), '-C', zeo_conf_path]
# )[0]
# self.path_list.append(wrapper)
return zeo_configuration_dict
def installRepozo(self, zodb_root_path):
"""
Add only repozo to cron (e.g. without tidstorage) allowing full
and incremental backups.
"""
backup_path = self.createBackupDirectory('zodb')
repozo_cron_path = os.path.join(self.cron_d, 'repozo')
repozo_cron_file = open(repozo_cron_path, 'w')
try:
repozo_cron_file.write('''
0 0 * * 0 %(repozo_binary)s --backup --full --file="%(zodb_root_path)s" --repository="%(backup_path)s"
0 * * * * %(repozo_binary)s --backup --file="%(zodb_root_path)s" --repository="%(backup_path)s"
''' % dict(repozo_binary=self.options['repozo_binary'],
zodb_root_path=zodb_root_path,
backup_path=backup_path))
finally:
repozo_cron_file.close()
self.path_list.append(repozo_cron_path)
def installTidStorage(self, ip, port, known_tid_storage_identifier_dict,
access_url):
"""Install TidStorage with all required backup tools
known_tid_storage_identifier_dict is a dictionary of:
(((ip, port),), storagename): (filestorage path, url for serialize)
url for serialize will be merged with access_url by internal tidstorage
"""
backup_base_path = self.createBackupDirectory('zodb')
# it is time to fill known_tid_storage_identifier_dict with backup
# destination
formatted_storage_dict = dict()
for key, v in known_tid_storage_identifier_dict.copy().iteritems():
# generate unique name for each backup
storage_name = key[1]
destination = os.path.join(backup_base_path, storage_name)
self._createDirectory(destination)
formatted_storage_dict[str(key)] = (v[0], destination, v[1])
logfile = os.path.join(self.log_directory, 'tidstorage.log')
pidfile = os.path.join(self.run_directory, 'tidstorage.pid')
statusfile = os.path.join(self.log_directory, 'tidstorage.tid')
timestamp_file_path = os.path.join(self.log_directory,
'repozo_tidstorage_timestamp.log')
# shared configuration file
tidstorage_config = self.createConfigurationFile('tidstorage.py',
self.substituteTemplate(self.getTemplateFilename('tidstorage.py.in'),
dict(
known_tid_storage_identifier_dict=pprint.pformat(formatted_storage_dict),
base_url='%s/%%s/serialize' % access_url,
host=ip,
port=port,
timestamp_file_path=timestamp_file_path,
logfile=logfile,
pidfile=pidfile,
statusfile=statusfile
)))
# TID server
tidstorage_server = zc.buildout.easy_install.scripts([('tidstoraged',
'slapos.recipe.librecipe.execute', 'execute')], self.ws, sys.executable,
self.wrapper_directory, arguments=[
self.options['tidstoraged_binary'], '--nofork', '--config',
tidstorage_config])[0]
self.registerLogRotation('tidsorage', [logfile, timestamp_file_path],
self.killpidfromfile + ' ' + pidfile + ' SIGHUP')
self.path_list.append(tidstorage_config)
self.path_list.append(tidstorage_server)
# repozo wrapper
tidstorage_repozo = zc.buildout.easy_install.scripts([('tidstorage_repozo',
'slapos.recipe.librecipe.execute', 'execute')], self.ws, sys.executable,
self.bin_directory, arguments=[
self.options['tidstorage_repozo_binary'], '--config', tidstorage_config,
'--repozo', self.options['repozo_binary'], '-z'])[0]
self.path_list.append(tidstorage_repozo)
# and backup configuration
tidstorage_repozo_cron = os.path.join(self.cron_d, 'tidstorage_repozo')
open(tidstorage_repozo_cron, 'w').write('''0 0 * * * %(tidstorage_repozo)s
0 0 * * * cp -f %(tidstorage_tid)s %(tidstorage_tid_backup)s'''%dict(
tidstorage_repozo=tidstorage_repozo,
tidstorage_tid=statusfile,
tidstorage_tid_backup=os.path.join(backup_base_path, 'tidstorage.tid')))
self.path_list.append(tidstorage_repozo_cron)
return dict(host=ip, port=port)
def _getApacheConfigurationDict(self, prefix, ip, port):
apache_conf = dict()
apache_conf['pid_file'] = os.path.join(self.run_directory,
prefix + '.pid')
apache_conf['lock_file'] = os.path.join(self.run_directory,
prefix + '.lock')
apache_conf['ip'] = ip
apache_conf['port'] = port
apache_conf['server_admin'] = 'admin@'
apache_conf['error_log'] = os.path.join(self.log_directory,
prefix + '-error.log')
apache_conf['access_log'] = os.path.join(self.log_directory,
prefix + '-access.log')
self.registerLogRotation(prefix, [apache_conf['error_log'],
apache_conf['access_log']], self.killpidfromfile + ' ' +
apache_conf['pid_file'] + ' SIGUSR1')
return apache_conf
def _writeApacheConfiguration(self, prefix, apache_conf, backend,
access_control_string=None):
rewrite_rule_template = \
"RewriteRule (.*) http://%(backend)s$1 [L,P]"
if access_control_string is None:
path_template = pkg_resources.resource_string(__name__,
'template/apache.zope.conf.path.in')
path = path_template % dict(path='/')
else:
path_template = pkg_resources.resource_string(__name__,
'template/apache.zope.conf.path-protected.in')
path = path_template % dict(path='/',
access_control_string=access_control_string)
d = dict(
path=path,
backend=backend,
backend_path='/',
port=apache_conf['port'],
vhname=path.replace('/', ''),
)
rewrite_rule = rewrite_rule_template % d
apache_conf.update(**dict(
path_enable=path,
rewrite_rule=rewrite_rule
))
apache_conf_string = pkg_resources.resource_string(__name__,
'template/apache.zope.conf.in') % apache_conf
return self.createConfigurationFile(prefix + '.conf', apache_conf_string)
def installFrontendZopeApache(self, ip, port, name, frontend_path, backend_url,
backend_path, key, certificate, access_control_string=None):
ident = 'frontend_' + name
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment
GitLab Nexedi Edition | About GitLab | About Nexedi | 沪ICP备2021021310号-2 | 沪ICP备2021021310号-7