diff --git a/component/apache-php/buildout.cfg b/component/apache-php/buildout.cfg index e6a684df6e17d16b1ae57466f5dae3276a44c8f8..6cd9bd78bcb2de8b693e54fd486e03b7fc91d978 100644 --- a/component/apache-php/buildout.cfg +++ b/component/apache-php/buildout.cfg @@ -15,7 +15,8 @@ extends = ../mariadb/buildout.cfg ../openldap/buildout.cfg ../pkgconfig/buildout.cfg - ../zip/buildout.cfg + ../zlib/buildout.cfg + [apache-php] # Note: Shall react on each build of apache and reinstall itself @@ -29,18 +30,16 @@ configure-options = --with-libxml-dir=${libxml2:location} --with-mysql=${mariadb:location} --with-zlib-dir=${zlib:location} - --with-bz2-dir=${bzip2:location} + --with-bz2=${bzip2:location} --with-mcrypt=${libmcrypt:location} --with-gd --with-jpeg-dir=${libjpeg:location} --with-png-dir=${libpng:location} --enable-gd-native-ttf - --with-ttf --with-freetype-dir=${freetype:location} --with-pdo-mysql=mysqlnd --with-mysqli=mysqlnd --with-curl=${curl:location} - --with-zip-dir=${zip:location} --with-imap=${cclient:location} --with-iconv-dir=${libiconv:location} --with-gettext=${gettext:location} @@ -52,10 +51,8 @@ configure-options = --enable-session --enable-exif --enable-zip - --enable-bz2 --enable-ftp - # Changing TMPDIR is required for PEAR installation. # It will create a pear/temp directory under the SR instead of a shared /tmp/pear/temp. # XXX we could mkdir tmp there @@ -69,23 +66,25 @@ environment = - [apache-php-postgres] -<=apache-php +# Note: Shall react on each build of apache and reinstall itself +recipe = hexagonit.recipe.cmmi +depends = + ${postgresql:url} +url = http://fr2.php.net/distributions/php-5.4.12.tar.bz2 +md5sum = 5c7b614242ae12e9cacca21c8ab84818 configure-options = --with-apxs2=${apache:location}/bin/apxs --with-libxml-dir=${libxml2:location} --with-zlib-dir=${zlib:location} - --with-bz2-dir=${bzip2:location} + --with-bz2=${bzip2:location} --with-mcrypt=${libmcrypt:location} --with-gd --with-jpeg-dir=${libjpeg:location} --with-png-dir=${libpng:location} --enable-gd-native-ttf - --with-ttf --with-freetype-dir=${freetype:location} --with-curl=${curl:location} - --with-zip-dir=${zip:location} --with-imap=${cclient:location} --with-iconv-dir=${libiconv:location} --with-gettext=${gettext:location} @@ -97,10 +96,21 @@ configure-options = --enable-session --enable-exif --enable-zip - --enable-bz2 --enable-ftp --with-pgsql=${postgresql:location} +# Changing TMPDIR is required for PEAR installation. +# It will create a pear/temp directory under the SR instead of a shared /tmp/pear/temp. +# XXX we could mkdir tmp there + +environment = + PKG_CONFIG_PATH=${libxml2:location}/lib/pkgconfig:${openssl:location}/lib/pkgconfig + PATH=${pkgconfig:location}/bin:${bzip2:location}/bin:${libxml2:location}/bin:%(PATH)s + LDFLAGS =-L${bzip2:location}/lib -Wl,-rpath -Wl,${bzip2:location}/lib -L${libtool:location}/lib -Wl,-rpath -Wl,${libtool:location}/lib -L${zlib:location}/lib -Wl,-rpath -Wl,${zlib:location}/lib -L${libmcrypt:location}/lib -Wl,-rpath -Wl,${libmcrypt:location}/libblkid + TMPDIR=${buildout:parts-directory}/${:_buildout_section_name_} + HOME=${apache:location} + + [libmcrypt] recipe = hexagonit.recipe.cmmi diff --git a/component/stunnel/buildout.cfg b/component/stunnel/buildout.cfg index b471e8f2e754f2df37d62516ed94ac941b8ee615..74776b1fb2f43deae0a2bc08305e7993c2b0d70e 100644 --- a/component/stunnel/buildout.cfg +++ b/component/stunnel/buildout.cfg @@ -17,8 +17,8 @@ filename = stunnel-4-hooks.py [stunnel-4] recipe = hexagonit.recipe.cmmi -url = ftp://ftp.stunnel.org/stunnel/archive/4.x/stunnel-4.54.tar.gz -md5sum = c2b1db99e3ed547214568959a8ed18ac +url = ftp://ftp.stunnel.org/stunnel/archive/4.x/stunnel-4.55.tar.gz +md5sum = ebe6670368d30c6a01ca33b9c50cb239 pre-configure-hook = ${stunnel-4-hook-download:location}/${stunnel-4-hook-download:filename}:pre_configure_hook configure-options = --enable-ipv6 diff --git a/slapos/recipe/apache_frontend/__init__.py b/slapos/recipe/apache_frontend/__init__.py index d097dcb9715ae95657936e18802facbc8481d08a..b2731dc06a8b53d47f57c197519dc7099e2c6257 100644 --- a/slapos/recipe/apache_frontend/__init__.py +++ b/slapos/recipe/apache_frontend/__init__.py @@ -85,8 +85,20 @@ class Recipe(BaseSlapRecipe): domain_dict = {} for slave_instance in slave_instance_list: + # Sanitize inputs backend_url = slave_instance.get("url", None) reference = slave_instance.get("slave_reference") + + if slave_instance.haskey("enable_cache"): + enable_cache = slave_instance.get("enable_cache", "").upper() in ('1', 'TRUE') + else: + enable_cache = False + + if slave_instance.haskey("type"): + slave_type = slave_instance.get("type", "").lower() + else: + slave_type = None + # Set scheme (http? https?) # Future work may allow to choose between http and https (or both?) scheme = 'http://' @@ -120,13 +132,15 @@ class Recipe(BaseSlapRecipe): slave_dict[reference] = "%s%s/" % (scheme, domain) # Check if we want varnish+stunnel cache. - if slave_instance.get("enable_cache", "").upper() in ('1', 'TRUE'): - # XXX-Cedric : need to refactor to clean code? (to many variables) - rewrite_rule = self.configureVarnishSlave( - base_varnish_port, backend_url, reference, service_dict, domain) - base_varnish_port += 2 - else: - rewrite_rule = "%s %s" % (domain, backend_url) + #if enable_cache: + # # XXX-Cedric : need to refactor to clean code? (to many variables) + # rewrite_rule = self.configureVarnishSlave( + # base_varnish_port, backend_url, reference, service_dict, domain) + # base_varnish_port += 2 + #else: + # rewrite_rule = "%s %s" % (domain, backend_url) + # Temporary forbid activation of cache until it is properly tested + rewrite_rule = "%s %s" % (domain, backend_url) # Finally, if successful, we add the rewrite rule to our list of rules if rewrite_rule: @@ -134,7 +148,7 @@ class Recipe(BaseSlapRecipe): # rule structure. # So we will have one RewriteMap for normal websites, and one # RewriteMap for Zope Virtual Host Monster websites. - if slave_instance.get("type", "").lower() in ['zope']: + if slave_type in ['zope']: rewrite_rule_zope_list.append(rewrite_rule) # For Zope, we have another dict containing the path e.g '/erp5/... rewrite_rule_path = "%s %s" % (domain, slave_instance.get('path', '')) @@ -529,7 +543,7 @@ class Recipe(BaseSlapRecipe): self._createDirectory(mod_ssl_cache_location) # Create "custom" apache configuration files if it does not exist. - # Note : Those files won't be erased or changed when slapgrid is ran. + # Note : Those files won't be erased or changed by slapgrid. # It can be freely customized by node admin. custom_apache_configuration_directory = os.path.join( self.data_root_directory, 'apache-conf.d') @@ -537,12 +551,14 @@ class Recipe(BaseSlapRecipe): # First one is included in the end of the apache configuration file custom_apache_configuration_file_location = os.path.join( custom_apache_configuration_directory, 'apache_frontend.custom.conf') - open(custom_apache_configuration_file_location, 'a') + if not os.path.exists(custom_apache_configuration_file_location): + open(custom_apache_configuration_file_location, 'w') # Second one is included in the virtualhost of apache configuration file custom_apache_virtual_configuration_file_location = os.path.join( custom_apache_configuration_directory, 'apache_frontend.virtualhost.custom.conf') - open(custom_apache_virtual_configuration_file_location, 'a') + if not os.path.exists(custom_apache_virtual_configuration_file_location): + open(custom_apache_virtual_configuration_file_location, 'w') # Create backup of custom apache configuration backup_path = self.createBackupDirectory('custom_apache_conf_backup') diff --git a/slapos/recipe/postgres/__init__.py b/slapos/recipe/postgres/__init__.py index 8dacb68b93f35a72457fdc34dee2f35a029949ab..465c1b82b6b8b447be01b4a00790988a48086bbc 100644 --- a/slapos/recipe/postgres/__init__.py +++ b/slapos/recipe/postgres/__init__.py @@ -154,7 +154,7 @@ class Recipe(GenericBaseRecipe): '# TYPE DATABASE USER ADDRESS METHOD', '', '# "local" is for Unix domain socket connections only (check unix_socket_permissions!)', - 'local all all ident', + 'local all all trust', 'host all all 127.0.0.1/32 md5', 'host all all ::1/128 md5', ] diff --git a/slapos/recipe/postgres/backup.py b/slapos/recipe/postgres/backup.py index 4341fe345d176b77b30f4ac48e1f1a3511605b18..7359358a1911337c26b9c710c236825f10d65d94 100644 --- a/slapos/recipe/postgres/backup.py +++ b/slapos/recipe/postgres/backup.py @@ -63,6 +63,7 @@ class ExportRecipe(GenericBaseRecipe): umask 077 %(bin)s/pg_dump \\ --host=%(pgdata-directory)s \\ + --username postgres \\ --format=custom \\ --file=%(backup-directory)s/database.dump \\ %(dbname)s @@ -102,6 +103,7 @@ class ImportRecipe(GenericBaseRecipe): #!/bin/sh %(bin)s/pg_restore \\ --host=%(pgdata-directory)s \\ + --username postgres \\ --dbname=%(dbname)s \\ --clean \\ --no-owner \\ diff --git a/software/apache-frontend/README.apache_frontend.txt b/software/apache-frontend/README.apache_frontend.txt index cac4c83ab04cae2c84daa39cedbb38b4c67dff7c..40046a52df8150e689ca0cd0199060654fde6989 100644 --- a/software/apache-frontend/README.apache_frontend.txt +++ b/software/apache-frontend/README.apache_frontend.txt @@ -43,6 +43,25 @@ all slave instances. Finally, the slave instance will be accessible from: https://someidentifier.moulefrite.org. + +How to have custom configuration in frontend server +=================================================== + +In your instance directory, you, as sysadmin, can directly edit two +configuration files that won't be overwritten by SlapOS to customize your +instance: + + * $PARTITION_PATH/srv/srv/apache-conf.d/apache_frontend.custom.conf + * $PARTITION_PATH/srv/srv/apache-conf.d/apache_frontend.virtualhost.custom.conf + +The first one is included in the end of the main apache configuration file. +The second one is included in the virtualhost of the main apache configuration file. + +SlapOS will jsut create those two files for you, then completely forget them. + +Note: make sure that the UNIX user of the instance has read access to those +files if you edit them. + Instance Parameters =================== diff --git a/software/maarch/instance-custom.cfg.in b/software/maarch/instance-custom.cfg.in index 2040e6661a78b1af7edc338d9d2025a936d683ef..eb749d905bff96b81ac357d28e99c333c06b8de4 100644 --- a/software/maarch/instance-custom.cfg.in +++ b/software/maarch/instance-custom.cfg.in @@ -3,17 +3,22 @@ [maarch-configuration] recipe = slapos.recipe.maarch:default htdocs = $${apache-php:htdocs} -db_host = $${postgres-urlparse:host} -db_port = $${postgres-urlparse:port} -db_dbname = $${postgres-urlparse:path} -db_username = $${postgres-urlparse:username} -db_password = $${postgres-urlparse:password} +db-host = $${postgres-urlparse:host} +db-port = $${postgres-urlparse:port} +db-dbname = $${postgres-urlparse:path} +db-username = $${postgres-urlparse:username} +db-password = $${postgres-urlparse:password} language = en -php_ini = $${directory:php-ini-dir}/php.ini -root_docservers = $${buildout:directory}/srv/docservers +php-ini = $${directory:php-ini-dir}/php.ini +root-docservers = $${buildout:directory}/srv/docservers dependency = $${apache-php:recipe} +maarch-sql-data-file = $${slap-parameter:maarch-sql-data-file} [publish-connection-informations] # XXX login should not be hardcoded login = superadmin -password = $${maarch-configuration:db_password} +password = $${maarch-configuration:db-password} + +[slap-parameter] +maarch-sql-data-file = + diff --git a/software/maarch/software.cfg b/software/maarch/software.cfg index 3f3aafb22d18bc94f7c96f0741436ea6cb18d38d..69862a98c23186f3280cbf339a06d6ba5c2ef5a2 100644 --- a/software/maarch/software.cfg +++ b/software/maarch/software.cfg @@ -5,11 +5,6 @@ extends = ../../stack/lapp/buildout.cfg -develop = - ${:parts-directory}/slapos.cookbook-repository - ${:parts-directory}/slapos.recipe.maarch-repository - - # += since we need rdiff-backup and friends parts += apache-php-postgres @@ -17,27 +12,19 @@ parts += eggs instance instance-apache-php - slapos.recipe.maarch-repository - check-recipe - - -[versions] -slapos.recipe.maarch = - + slapos-recipe-maarch-egg -[slapos.recipe.maarch-repository] -recipe = slapos.recipe.build:gitclone -repository = http://git.erp5.org/repos/slapos.recipe.maarch.git -branch = master -git-executable = ${git:location}/bin/git -[check-recipe] -recipe = plone.recipe.command -stop-on-error = true -update-command = ${:command} -command = - grep parts ${buildout:develop-eggs-directory}/slapos.recipe.maarch.egg-link +#---------------- +#-- +#-- Explicitly provide the configuration egg, +#-- otherwise instance.cfg won't be able to use it. +#-- +[slapos-recipe-maarch-egg] +recipe = zc.recipe.egg +eggs = + slapos.recipe.maarch #---------------- @@ -63,7 +50,7 @@ part-list = maarch-configuration recipe = slapos.recipe.template url = ${:_profile_base_location_}/instance-custom.cfg.in output = ${buildout:directory}/instance-custom.cfg -md5sum = d1b02a8c571886a225f2f1c8da06286c +md5sum = f86b311d443156e327a1b5f5acfb22b1 mode = 0644 @@ -83,3 +70,6 @@ install_cmd = #---------------- +[versions] +slapos.recipe.maarch = 0.4 + diff --git a/stack/erp5/buildout.cfg b/stack/erp5/buildout.cfg index 46e55a1c18a87df00c6c3ad4c802eba3766ef851..8594d92d18c1d0789e31c6929ab159abbe180d0c 100644 --- a/stack/erp5/buildout.cfg +++ b/stack/erp5/buildout.cfg @@ -141,7 +141,7 @@ parts = [slapos.cookbook-repository] recipe = slapos.recipe.build:gitclone repository = http://git.erp5.org/repos/slapos.git -branch = erp5-component +branch = erp5 git-executable = ${git:location}/bin/git [check-recipe] @@ -346,7 +346,7 @@ repository_id_list = erp5 [erp5] recipe = slapos.recipe.build:gitclone repository = http://git.erp5.org/repos/erp5.git -branch = erp5-component +branch = master git-executable = ${git:location}/bin/git [fix-products-paths] diff --git a/stack/lamp/apache/instance-apache-export.cfg.jinja2 b/stack/lamp/apache/instance-apache-export.cfg.jinja2 index ac2c24505c5d86beb50fbe4e49e25dab7fcfe998..594faa18e9f9b684140d2c07db7bb39e3a65ef48 100644 --- a/stack/lamp/apache/instance-apache-export.cfg.jinja2 +++ b/stack/lamp/apache/instance-apache-export.cfg.jinja2 @@ -29,9 +29,13 @@ parts += {{ replicated.replicate("mariadb", "3", "mariadb-export", "mariadb-import") }} -# Nothing to do for the exporter. Just dummy part that does nothing. -# For httpd instance, PBS will directly pull data from srv/www. -# XXX-Cedric: write a real backup system. +# Nothing to do for the exporter. This wrapper is intended +# to produce the "dump" files that have to be backed up. +# So, in case of binary DB data, we would back up the ASCII dump +# files, or a similar data format that can be restored on any machine +# (i.e. postgres 'custom') +# In the case of an httpd instance, the src/www directory can be directly +# pulled from the PBS, we don't need to prepare anything. [exporter] wrapper = /bin/true diff --git a/stack/lapp/apache/instance-apache-backup.cfg.in b/stack/lapp/apache/instance-apache-backup.cfg.in deleted file mode 100644 index ffe3b4fc7d14c61682f90d6c0cb7927857dfaec5..0000000000000000000000000000000000000000 --- a/stack/lapp/apache/instance-apache-backup.cfg.in +++ /dev/null @@ -1,197 +0,0 @@ -[buildout] - -parts = - urls - apache-proxy - logrotate - logrotate-entry-apache - cron - cron-entry-logrotate - sshkeys-authority - sshkeys-dropbear - dropbear-server - dropbear-server-pbs-authorized-key - -eggs-directory = ${buildout:eggs-directory} -develop-eggs-directory = ${buildout:develop-eggs-directory} -offline = true - - -#---------------- -#-- -#-- Creation of all needed directories. - -[rootdirectory] -recipe = slapos.cookbook:mkdirectory -etc = $${buildout:directory}/etc -var = $${buildout:directory}/var -srv = $${buildout:directory}/srv -bin = $${buildout:directory}/bin -tmp = $${buildout:directory}/tmp - -[basedirectory] -recipe = slapos.cookbook:mkdirectory -log = $${rootdirectory:var}/log -services = $${rootdirectory:etc}/service -run = $${rootdirectory:var}/run -backup = $${rootdirectory:srv}/backup -promises = $${rootdirectory:etc}/promise - -[directory] -recipe = slapos.cookbook:mkdirectory -htdocs = $${rootdirectory:srv}/www -logrotate-entries = $${rootdirectory:etc}/logrotate.d -logrotate-backup = $${basedirectory:backup}/logrotate -cronstamps = $${rootdirectory:etc}/cronstamps -cron-entries = $${rootdirectory:etc}/cron.d -crontabs = $${rootdirectory:etc}/crontabs -ssh = $${rootdirectory:etc}/ssh -sshkeys = $${rootdirectory:srv}/sshkeys -httpd-log = $${basedirectory:log}/apache - - -#---------------- -#-- -#-- Deploy cron. - -[cron] -recipe = slapos.cookbook:cron -dcrond-binary = ${dcron:location}/sbin/crond -cron-entries = $${directory:cron-entries} -crontabs = $${directory:crontabs} -cronstamps = $${directory:cronstamps} -catcher = $${cron-simplelogger:wrapper} -binary = $${basedirectory:services}/crond - -[cron-simplelogger] -recipe = slapos.cookbook:simplelogger -wrapper = $${rootdirectory:bin}/cron_simplelogger -log = $${basedirectory:log}/crond.log - - -#---------------- -#-- -#-- Deploy logrotate. - -[cron-entry-logrotate] -<= cron -recipe = slapos.cookbook:cron.d -name = logrotate -frequency = 0 0 * * * -command = $${logrotate:wrapper} - -[logrotate] -recipe = slapos.cookbook:logrotate -# Binaries -logrotate-binary = ${logrotate:location}/usr/sbin/logrotate -gzip-binary = ${gzip:location}/bin/gzip -gunzip-binary = ${gzip:location}/bin/gunzip -# Directories -wrapper = $${rootdirectory:bin}/logrotate -conf = $${rootdirectory:etc}/logrotate.conf -logrotate-entries = $${directory:logrotate-entries} -backup = $${directory:logrotate-backup} -state-file = $${rootdirectory:srv}/logrotate.status - - -#---------------- -#-- -#-- sshkeys - -[sshkeys-directory] -recipe = slapos.cookbook:mkdirectory -requests = $${directory:sshkeys}/requests -keys = $${directory:sshkeys}/keys - -[sshkeys-authority] -recipe = slapos.cookbook:sshkeys_authority -request-directory = $${sshkeys-directory:requests} -keys-directory = $${sshkeys-directory:keys} -wrapper = $${basedirectory:services}/sshkeys_authority -keygen-binary = ${dropbear:location}/bin/dropbearkey - -[sshkeys-dropbear] -<= sshkeys-authority -recipe = slapos.cookbook:sshkeys_authority.request -name = dropbear -type = rsa -executable = $${dropbear-server:wrapper} -public-key = $${dropbear-server:rsa-keyfile}.pub -private-key = $${dropbear-server:rsa-keyfile} -wrapper = $${basedirectory:services}/sshd - - -#---------------- -#-- -#-- Dropbear. - -[dropbear-server] -recipe = slapos.cookbook:dropbear -host = $${slap-network-information:global-ipv6} -port = 2222 -home = $${directory:ssh} -wrapper = $${rootdirectory:bin}/raw_sshd -shell = $${rdiff-backup-server:wrapper} -rsa-keyfile = $${directory:ssh}/server_key.rsa -dropbear-binary = ${dropbear:location}/sbin/dropbear - -[dropbear-server-pbs-authorized-key] -<= dropbear-server -recipe = slapos.cookbook:dropbear.add_authorized_key -key = $${slap-parameter:authorized-key} - - -#---------------- -#-- -#-- rdiff - -[rdiff-backup-server] -recipe = slapos.cookbook:pbs -client = false -path = $${directory:htdocs} -wrapper = $${rootdirectory:bin}/rdiffbackup-server -rdiffbackup-binary = ${buildout:bin-directory}/rdiff-backup - - -#---------------- -#-- -#-- Apache Proxy. - -[apache-proxy] -recipe = slapos.cookbook:apacheproxy -url = $${slap-parameter:proxy-url} -pid-file = $${basedirectory:run}/apache.pid -lock-file = $${basedirectory:run}/apache.lock -ip = $${slap-network-information:global-ipv6} -port = 8080 -error-log = $${directory:httpd-log}/error.log -access-log = $${directory:httpd-log}/access.log -httpd-conf = $${rootdirectory:etc}/apache.conf -wrapper = $${basedirectory:services}/apache - -promise = $${basedirectory:promises}/apache - -httpd-binary = ${apache:location}/bin/httpd - -[logrotate-entry-apache] -<= logrotate -recipe = slapos.cookbook:logrotate.d -name = apache -log = $${apache-proxy:error-log} $${apache-proxy:access-log} -frequency = daily -rotate-num = 30 -sharedscripts = true -notifempty = true -create = true - - -#---------------- -#-- -#-- Publish instance parameters. - -[urls] -recipe = slapos.cookbook:publish -url = http://[$${apache-proxy:ip}]:$${apache-proxy:port}/ -ssh-public-key = $${sshkeys-dropbear:public-key-value} -ssh-url = ssh://nobody@[$${dropbear-server:host}]:$${dropbear-server:port}/$${rdiff-backup-server:path} - diff --git a/stack/lapp/apache/instance-apache-export.cfg.jinja2 b/stack/lapp/apache/instance-apache-export.cfg.jinja2 new file mode 100644 index 0000000000000000000000000000000000000000..cbfea109d5beadc47dc81c317e95f106c8812b41 --- /dev/null +++ b/stack/lapp/apache/instance-apache-export.cfg.jinja2 @@ -0,0 +1,58 @@ +# This file is responsible of three things: +# 1/ Act as "Apache exporter" +# 2/ Act as "Postgres backup infrastructure requester" +# 3/ Act as "Apache" instance + +{% import 'parts' as parts %} +{% import 'replicated' as replicated %} + +[buildout] +extends = {{templateapache}} + {{templatepbsreadyexport}} + +parts += + {{ parts.replicate("postgres", "3") }} + +# Repeating parts from instance-apache-php. +# XXX-Cedric: how to simplify this? + certificate-authority + ca-stunnel + logrotate + logrotate-entry-apache + logrotate-entry-stunnel + cron + cron-entry-logrotate + promise + frontend-promise + content-promise + publish-connection-informations + +{{ replicated.replicate("postgres", "3", "postgres-export", "postgres-import") }} + +# Nothing to do for the exporter. This wrapper is intended +# to produce the "dump" files that have to be backed up. +# So, in case of binary DB data, we would back up the ASCII dump +# files, or a similar data format that can be restored on any machine +# (i.e. postgres 'custom') +# In the case of an httpd instance, the src/www directory can be directly +# pulled from the PBS, we don't need to prepare anything. +[exporter] +wrapper = /bin/true + +# State that we want to backup srv/www directory, not srv/backup. +# XXX-Cedric: works well, but doesn't work with big data. +[rdiff-backup-server] +path = ${directory:www} + +# Add "exporter" parameters to list of published connection parameters +[publish-connection-informations] +# XXX-Cedric: Long term goal: could be a recipe that requests an instance and +# bubbles ALL +# parameters of the requested instance. Requirement: aggregated publish. +<= resilient-publish-connection-parameter + +# XXX-Cedric: resilient overwrites what's returned from request-postgres +# XXX-Cedric: change the request method to return everything from +# getConnectionParameterDict() +[request-postgres] +return = ssh-public-key ssh-url notification-id ip url diff --git a/stack/lapp/apache/instance-apache-import.cfg.in b/stack/lapp/apache/instance-apache-import.cfg.in new file mode 100644 index 0000000000000000000000000000000000000000..009a48922ee045ab2ffbb4fd16746d0b511cbd68 --- /dev/null +++ b/stack/lapp/apache/instance-apache-import.cfg.in @@ -0,0 +1,20 @@ +[buildout] +eggs-directory = ${buildout:eggs-directory} +develop-eggs-directory = ${buildout:develop-eggs-directory} +offline = true + +extends = ${pbsready-import:output} + +[directory] +srv = $${buildout:directory}/srv +www = $${:srv}/www/ + +# Nothing to do for the import. Just dummy part that does nothing. +# For httpd instance, PBS will directly push data to srv/www. +# XXX-Cedric: write a real backup system. +[importer] +wrapper = /bin/true + +# State that we want to push to srv/www directory, not srv/backup. +[rdiff-backup-server] +path = $${directory:www} diff --git a/stack/lapp/apache/instance-apache-php.cfg.in b/stack/lapp/apache/instance-apache-php.cfg.in index 5521da7e8b0f3cb27178d2a8807a1f7fb0205e5f..cd425c71109592d3dc27f1b896467a68c8138ab9 100644 --- a/stack/lapp/apache/instance-apache-php.cfg.in +++ b/stack/lapp/apache/instance-apache-php.cfg.in @@ -57,6 +57,7 @@ logrotate-backup = $${basedirectory:backup}/logrotate report = $${rootdirectory:etc}/report stunnel-conf = $${rootdirectory:etc}/stunnel xml-report = $${rootdirectory:var}/xml_report +www = $${rootdirectory:srv}/www/ [cadirectory] recipe = slapos.cookbook:mkdirectory @@ -195,7 +196,7 @@ url = $${request-postgres:connection-url} recipe = slapos.cookbook:apachephp source = ${application:location} -htdocs = $${rootdirectory:srv}/www/ +htdocs = $${directory:www} pid-file = $${basedirectory:run}/apache.pid lock-file = $${basedirectory:run}/apache.lock ip = $${slap-network-information:global-ipv6} diff --git a/stack/lapp/buildout.cfg b/stack/lapp/buildout.cfg index b642c4b228450b9c6516f6aa73f950f560c28d38..673f746e7b43172a23ab34fab41eec2b8ab0d58e 100644 --- a/stack/lapp/buildout.cfg +++ b/stack/lapp/buildout.cfg @@ -1,4 +1,7 @@ [buildout] + +ignore-existing = true + parts = slapos-cookbook apache-php-postgres @@ -15,11 +18,14 @@ parts = instance-postgres-import instance-postgres-export +#Contains the importer and exporter recipes for apache + instance-apache-import + instance-apache-export + extends = - ../resilient/buildout.cfg - ../../component/apache/buildout.cfg ../../component/apache-php/buildout.cfg + ../../component/apache/buildout.cfg ../../component/dash/buildout.cfg ../../component/dcron/buildout.cfg ../../component/gzip/buildout.cfg @@ -29,6 +35,7 @@ extends = ../../component/rdiff-backup/buildout.cfg ../../component/stunnel/buildout.cfg ../../component/dropbear/buildout.cfg + ../resilient/buildout.cfg ../slapos.cfg @@ -48,21 +55,35 @@ strip-top-level-dir = true recipe = slapos.recipe.template url = ${:_profile_base_location_}/instance.cfg.in output = ${buildout:directory}/instance.cfg -md5sum = a482fa0e72839b4bd75b169ac1460d64 +md5sum = 25d07b5101d5f566398686642ada4cee mode = 0644 [instance-apache-php] recipe = slapos.recipe.template url = ${:_profile_base_location_}/apache/instance-apache-php.cfg.in output = ${buildout:directory}/instance-apache-php.cfg -md5sum = 72b70452d1c077cfcd0f268181506b8e +md5sum = 823257dda6f3068a38c6b69c771cf307 mode = 0644 -[instance-apache-backup] +[instance-apache-import] recipe = slapos.recipe.template -url = ${:_profile_base_location_}/apache/instance-apache-backup.cfg.in -output = ${buildout:directory}/instance-apache-backup.cfg -md5sum = db879141c0b6a77ef8b3b7e699f5583a +url = ${:_profile_base_location_}/apache/instance-apache-import.cfg.in +output = ${buildout:directory}/instance-apache-import.cfg +md5sum = f1dc2a71d362b5d2d36481ffefdd2293 +mode = 0644 + +[instance-apache-export] +recipe = slapos.recipe.template:jinja2 +template = ${:_profile_base_location_}/apache/instance-apache-export.cfg.jinja2 +rendered = ${buildout:directory}/instance-apache-export.cfg + +context = key templateapache instance-apache-php:output + key templatepbsreadyexport pbsready-export:output + +import-list = file parts template-parts:destination + file replicated template-replicated:destination + +md5sum = bdc7e126567ece6bf93a9bb493e29fac mode = 0644 [instance-resilient] @@ -70,16 +91,15 @@ recipe = slapos.recipe.template:jinja2 template = ${:_profile_base_location_}/instance-resilient.cfg.jinja2 rendered = ${buildout:directory}/instance-resilient.cfg -context = key templateapache instance-apache-php:output - key dropbear dropbear:location - key buildout buildout:bin-directory +context = key buildout buildout:bin-directory + key develop_eggs_directory buildout:develop-eggs-directory + key eggs_directory buildout:eggs-directory import-list = file parts template-parts:destination file replicated template-replicated:destination -md5sum = 46c7d8f691bd37d84e0bd03b83e51d14 +md5sum = ef38aa9810ce20960382261f235abfcd mode = 0644 - [instance-postgres] recipe = slapos.recipe.template @@ -99,7 +119,7 @@ mode = 0644 recipe = slapos.recipe.template url = ${:_profile_base_location_}/postgres/instance-postgres-export.cfg.in output = ${buildout:directory}/instance-postgres-export.cfg -md5sum = 68080e5d861eb3474442211dd65c668b +md5sum = d95205a5fc2825e9709ed6db295111e2 mode = 0644 @@ -155,7 +175,6 @@ eggs = ${psycopg2:egg} slapos.toolbox - [networkcache] # Romain Courteaud + Sebastien Robin + Alain Takoudjou # + Cedric de Saint Martin signature certificate diff --git a/stack/lapp/instance-resilient.cfg.jinja2 b/stack/lapp/instance-resilient.cfg.jinja2 index 022a37f951fb6ab88e577fd9e08f6ba8b69ce09d..0ec907a442b5543b1b7a97adbb56b3c68e03374d 100644 --- a/stack/lapp/instance-resilient.cfg.jinja2 +++ b/stack/lapp/instance-resilient.cfg.jinja2 @@ -4,172 +4,47 @@ {% import 'replicated' as replicated %} [buildout] - -extends = - {{templateapache}} +eggs-directory = {{ eggs_directory }} +develop-eggs-directory = {{ develop_eggs_directory }} +offline = true # += because we need to take up parts (like instance-custom, slapmonitor etc) from the profile we extended parts += - {{ parts.replicate("postgres","3") }} - request-apache-backup-1 - request-apache-backup-2 - - request-pull-backup-server-apache-1 - request-pull-backup-server-apache-backup-1 - - request-pull-backup-server-apache-2 - request-pull-backup-server-apache-backup-2 - + {{ parts.replicate("apache", "3") }} publish-connection-informations - apache-php - stunnel - certificate-authority - ca-stunnel - logrotate - logrotate-entry-apache - logrotate-entry-stunnel - cron - cron-entry-logrotate - dropbear-server - sshkeys-authority - dropbear-server-pbs-authorized-key - - request-pull-backup-server - -{{ replicated.replicate("postgres", "3", "postgres-export", "postgres-import") }} - - -[request-pull-backup-server] -<= request-pbs-common -name = PBS (Pull Backup Server) -return = ssh-key notification-url feeds-url -slave = false - -[sshkeys-directory] -recipe = slapos.cookbook:mkdirectory -requests = ${directory:sshkeys}/requests -keys = ${directory:sshkeys}/keys - -[sshkeys-authority] -recipe = slapos.cookbook:sshkeys_authority -request-directory = ${sshkeys-directory:requests} -keys-directory = ${sshkeys-directory:keys} -wrapper = ${basedirectory:services}/sshkeys_authority -keygen-binary = {{dropbear}}/bin/dropbearkey - -[sshkeys-dropbear] -<= sshkeys-authority -recipe = slapos.cookbook:sshkeys_authority.request -name = dropbear -type = rsa -executable = ${dropbear-server:wrapper} -public-key = ${dropbear-server:rsa-keyfile}.pub -private-key = ${dropbear-server:rsa-keyfile} -wrapper = ${basedirectory:services}/sshd - -[dropbear-server] -recipe = slapos.cookbook:dropbear -host = ${slap-network-information:global-ipv6} -port = 2222 -home = ${directory:ssh} -wrapper = ${rootdirectory:bin}/raw_sshd -shell = ${rdiff-backup-server:wrapper} -rsa-keyfile = ${directory:ssh}/server_key.rsa -dropbear-binary = {{dropbear}}/sbin/dropbear - -[dropbear-server-pbs-authorized-key] -<= dropbear-server -recipe = slapos.cookbook:dropbear.add_authorized_key -key = ${request-pull-backup-server:connection-ssh-key} - -[rdiff-backup-server] -<= apache-php -recipe = slapos.cookbook:pbs -client = false -path = ${apache-php:htdocs} -wrapper = ${rootdirectory:bin}/rdiffbackup-server -rdiffbackup-binary = {{buildout}}/rdiff-backup - -[request-apache-backup-1] -<= slap-connection -recipe = slapos.cookbook:request -name = Apache Backup 1 -software-url = ${slap-connection:software-release-url} -software-type = apache-backup -return = url ssh-url ssh-public-key -config = authorized-key proxy-url -config-authorized-key = ${request-pull-backup-server:connection-ssh-key} -config-proxy-url = ${publish-connection-informations:url} - -[request-apache-backup-2] -<= slap-connection -recipe = slapos.cookbook:request -name = Apache Backup 2 -software-url = ${slap-connection:software-release-url} -software-type = apache-backup -return = url ssh-url ssh-public-key -config = authorized-key proxy-url -config-authorized-key = ${request-pull-backup-server:connection-ssh-key} -config-proxy-url = ${publish-connection-informations:url} - -[request-pull-backup-server-apache-1] -<= request-pbs-common -name = PBS pulling from Apache 1 -config = url name type server-key notify notification-id frequency -config-url = ssh://nobody@[${dropbear-server:host}]:${dropbear-server:port}/${rdiff-backup-server:path} -config-name = ${slap-connection:computer-id}-${slap-connection:partition-id}-apache -config-type = pull -config-server-key = ${sshkeys-dropbear:public-key-value} -config-notify = ${request-pull-backup-server:connection-notification-url} -config-notification-id = ${slap-connection:computer-id}-${slap-connection:partition-id}-apache-pull -config-frequency = 30 * * * * -slave = true -sla = instance_guid -sla-instance_guid = ${request-pull-backup-server:instance_guid} - -[request-pull-backup-server-apache-2] -<= request-pbs-common -name = PBS pulling from Apache 2 -config = url name type server-key notify notification-id frequency -config-url = ssh://nobody@[${dropbear-server:host}]:${dropbear-server:port}/${rdiff-backup-server:path} -config-name = ${slap-connection:computer-id}-${slap-connection:partition-id}-apache -config-type = pull -config-server-key = ${sshkeys-dropbear:public-key-value} -config-notify = ${request-pull-backup-server:connection-notification-url} -config-notification-id = ${slap-connection:computer-id}-${slap-connection:partition-id}-apache-pull -config-frequency = 30 * * * * -slave = true -sla = instance_guid -sla-instance_guid = ${request-pull-backup-server:instance_guid} - - -[request-pull-backup-server-apache-backup-1] -<= request-pbs-common -name = PBS pushing to ${request-apache-backup-1:name} -config = url name type server-key on-notification -config-url = ${request-apache-backup-1:connection-ssh-url} -config-name = ${request-pull-backup-server-apache-1:config-name} -config-type = push -config-server-key = ${request-apache-backup-1:connection-ssh-public-key} -config-on-notification = ${request-pull-backup-server:connection-feeds-url}${request-pull-backup-server-apache-1:config-notification-id} -slave = true -sla = instance_guid -sla-instance_guid = ${request-pull-backup-server:instance_guid} - -[request-pull-backup-server-apache-backup-2] -<= request-pbs-common -name = PBS pushing to ${request-apache-backup-2:name} -config = url name type server-key on-notification -config-url = ${request-apache-backup-2:connection-ssh-url} -config-name = ${request-pull-backup-server-apache-2:config-name} -config-type = push -config-server-key = ${request-apache-backup-2:connection-ssh-public-key} -config-on-notification = ${request-pull-backup-server:connection-feeds-url}${request-pull-backup-server-apache-2:config-notification-id} -slave = true -sla = instance_guid -sla-instance_guid = ${request-pull-backup-server:instance_guid} - -[directory] -ssh = ${rootdirectory:etc}/ssh -sshkeys = ${rootdirectory:srv}/sshkeys +{{ replicated.replicate("apache", "3", "apache-export", "apache-import") }} + +# Bubble up the parameters +[request-apache] +return = url ssh-public-key ssh-url notification-id ip url backend_url +# XXX: hardcoded values +config = domain number authorized-key notify ip-list namebase postgres1-computer-guid pbs-postgres1-computer-guid postgres2-computer-guid pbs-postgres2-computer-guid postgres3-computer-guid pbs-postgres3-computer-guid maarch-sql-data-file +config-postgres1-computer-guid = ${slap-parameter:postgres1-computer-guid} +config-pbs-postgres1-computer-guid = ${slap-parameter:pbs-postgres1-computer-guid} +config-postgres2-computer-guid = ${slap-parameter:postgres2-computer-guid} +config-pbs-postgres2-computer-guid = ${slap-parameter:pbs-postgres2-computer-guid} +config-postgres3-computer-guid = ${slap-parameter:postgres3-computer-guid} +config-pbs-postgres3-computer-guid = ${slap-parameter:pbs-postgres3-computer-guid} +config-domain = ${slap-parameter:domain} +config-maarch-sql-data-file = ${slap-parameter:maarch-sql-data-file} + +[publish-connection-informations] +recipe = slapos.cookbook:publish +backend_url = ${request-apache:connection-backend_url} +url = ${request-apache:connection-url} + +[slap-parameter] +# Default parameters for distributed deployment +# I.e state "backup1 of postgres should go there, ..." +# XXX-Cedric: Hardcoded number of backups. Should be dynamically generated. +postgres1-computer-guid = +pbs-postgres1-computer-guid = +postgres2-computer-guid = +pbs-postgres2-computer-guid = +postgres3-computer-guid = +pbs-postgres3-computer-guid = +# XXX-Cedric: Hardcoded parameters. Should be dynamically generated. +domain = +# the following parameters are specific to maarch +maarch-sql-data-file = diff --git a/stack/lapp/instance.cfg.in b/stack/lapp/instance.cfg.in index 596a9dfb03c9981c7e2aefb3e0c17f4d56e0f216..910502c6647e81ac1208ade4978f18f4decacaa6 100644 --- a/stack/lapp/instance.cfg.in +++ b/stack/lapp/instance.cfg.in @@ -14,7 +14,8 @@ resilient = ${instance-resilient:rendered} postgres = ${instance-postgres:output} postgres-import = ${instance-postgres-import:output} postgres-export = ${instance-postgres-export:output} -apache-backup = ${instance-apache-backup:output} +apache-import = ${instance-apache-import:output} +apache-export = ${instance-apache-export:rendered} #frozen creates a syntax error, meaning it can keep its data. #It's dirty as hell, it needs to be replaced. diff --git a/stack/lapp/postgres/instance-postgres-export.cfg.in b/stack/lapp/postgres/instance-postgres-export.cfg.in index 4bce336d727addf488fa1b69c820675c753b5c81..f41f69f5ed306262dc75ae9b50ab513aa989114e 100644 --- a/stack/lapp/postgres/instance-postgres-export.cfg.in +++ b/stack/lapp/postgres/instance-postgres-export.cfg.in @@ -19,3 +19,8 @@ bin = $${postgres-instance:bin} pgdata-directory = $${postgres-instance:pgdata-directory} backup-directory = $${postgres-instance:backup-directory} dbname = $${postgres-instance:dbname} + + +# Extends publish section with resilient parameters +[urls] +<= resilient-publish-connection-parameter