Commit 09162bd3 authored by Jérome Perrin's avatar Jérome Perrin

PALO: polished ETL and support for ssl

parent 7559a94a
[buildout] [buildout]
parts = parts =
instance-parameter instance-parameter
directory
palo_olap palo_olap
publish-connection-parameter
palo_etl palo_etl
publish-connection-parameter
eggs-directory = ${buildout:eggs-directory} eggs-directory = ${buildout:eggs-directory}
develop-eggs-directory = ${buildout:develop-eggs-directory} develop-eggs-directory = ${buildout:develop-eggs-directory}
offline = true offline = true
# TODO: json schema with parameters
# TODO: review certificate generation, file permissions and passwords
# common parts
[instance-parameter] [instance-parameter]
recipe = slapos.cookbook:slapconfiguration recipe = slapos.cookbook:slapconfiguration
computer = $${slap_connection:computer_id} computer = $${slap_connection:computer_id}
...@@ -19,51 +21,53 @@ url = $${slap_connection:server_url} ...@@ -19,51 +21,53 @@ url = $${slap_connection:server_url}
key = $${slap_connection:key_file} key = $${slap_connection:key_file}
cert = $${slap_connection:cert_file} cert = $${slap_connection:cert_file}
# erp5_url is the URL of an ERP5 instance, with erp5_palo business template installed
configuration.erp5_url = configuration.erp5_url =
# TODO: configuration of a simple password based authentication ?
[palo_olap_parameter]
ipv4 = $${instance-parameter:ipv4-random} [publish-connection-parameter]
ipv6 = $${instance-parameter:ipv6-random} recipe = slapos.cookbook:publish
palo_olap_port = 7777 palo_olap = [$${palo_olap_parameter:ipv6}]:$${palo_olap_parameter:palo_olap_port}
palo_olap_admin_port = 7778 palo_olap_ssl = [$${palo_olap_parameter:ipv6}]:$${palo_olap_parameter:palo_olap_ssl_port}
palo_etl_url = https://[$${palo_etl_parameter:tomcat_host}]:$${palo_etl_parameter:tomcat_port}/etlserver/services//ETL-Server?wsdl
# palo_olap_admin_url = [$${palo_olap_parameter:ipv6}]:$${palo_olap_parameter:palo_olap_admin_port}
[directory] [directory]
recipe = slapos.cookbook:mkdirectory recipe = slapos.cookbook:mkdirectory
home = $${buildout:directory} home = $${buildout:directory}
bin = $${:home}/bin
etc = $${:home}/etc etc = $${:home}/etc
var = $${:home}/var script = $${:etc}/run
script = $${:etc}/run/
service = $${:etc}/service service = $${:etc}/service
promise = $${:etc}/promise/ promise = $${:etc}/promise
var = $${:home}/var
log = $${:var}/log log = $${:var}/log
srv = $${:home}/srv
[directory_palo_etl]
<= directory
data_directory = $${:var}/palo_olap/
[directory_tomcat]
<= directory
catalina_base = $${:var}/palo_etl
catalina_logs = $${:catalina_base}/logs
catalina_temp = $${:catalina_base}/temp
catalina_webapps = $${:catalina_base}/webapps
catalina_work = $${:catalina_base}/work
catalina_conf = $${:catalina_base}/conf
# Palo olap instance
[palo_olap_parameter]
ipv4 = $${instance-parameter:ipv4-random}
ipv6 = $${instance-parameter:ipv6-random}
palo_olap_port = 7777
palo_olap_ssl_port = 7778
palo_olap_admin_port = 7779
key-file = $${certificate_palo_olap:key-file}
dh1024-file = $${certificate_palo_olap:dh1024-file}
[TODO] # XXX this depends on architecture. Maybe we need to patch palo_olap for that
todo = extensions_dir = ${palo_olap:location}/usr/lib64
tunnel
etl
log_rotation
[directory_palo_olap]
<= directory
data_directory = $${:var}/palo_olap
[palo_ini] [palo_ini]
recipe = slapos.recipe.template:jinja2 recipe = slapos.recipe.template:jinja2
template = ${:_profile_base_location_}/palo.ini.in template = ${template_palo_ini:target}
rendered = $${directory:etc}/palo.ini rendered = $${directory:etc}/palo.ini
extensions = jinja2.ext.do context =
context = import json_module json
key directory_log directory:log key directory_log directory:log
raw palo_olap_repository_location ${palo_olap-repository.git:location} raw palo_olap_repository_location ${palo_olap-repository.git:location}
section parameter instance-parameter section parameter instance-parameter
...@@ -71,45 +75,69 @@ context = import json_module json ...@@ -71,45 +75,69 @@ context = import json_module json
key erp5_url instance-parameter:configuration.erp5_url key erp5_url instance-parameter:configuration.erp5_url
raw erp5_login_worker_path ${erp5_login_worker:output} raw erp5_login_worker_path ${erp5_login_worker:output}
[palo_olap] [certificate_palo_olap]
recipe = slapos.cookbook:wrapper
command-line = ${palo_olap:location}/usr/bin/palo --data-directory $${directory_palo_etl:data_directory} --init-file $${palo_ini:rendered}
wrapper-path = $${directory:service}/palo_olap
[tomcat_palo_etl]
recipe = plone.recipe.command recipe = plone.recipe.command
command = echo "needed ?" # Manually generate certificates as documented in palo.ini.sample
command =
${openssl:location}/bin/openssl req -x509 -nodes -days 3650 \
-subj "/C=AA/ST=X/L=X/O=Dis/CN=$${palo_olap_parameter:ipv6}" \
-newkey rsa:1024 -keyout $${:key-file} -out $${:key-file} && \
${openssl:location}/bin/openssl dhparam -2 -outform PEM -out $${:dh1024-file} 1024
key-file = $${directory:etc}/palo_olap.pem
dh1024-file = $${directory:etc}/dh1024.pem
[palo_etl] [palo_olap]
recipe = slapos.cookbook:wrapper recipe = slapos.cookbook:wrapper
command-line = ${tomcat:location}/bin/catalina.sh run command-line = ${palo_olap:location}/usr/bin/palo --data-directory $${directory_palo_olap:data_directory} --init-file $${palo_ini:rendered}
wrapper-path = $${directory:service}/palo_etl wrapper-path = $${directory:service}/palo_olap
environment = JAVA_HOME = ${java:location}
CATALINA_BASE = $${directory_tomcat:catalina_base}
dependencies = $${palo_etl_server_xml:rendered} $${tomcat_palo_etl:recipe}
[palo_etl_server_xml]
recipe = slapos.recipe.template:jinja2
# XXX template = ${template_server_xml:location}/${template_server_xml:filename}
template = ${template_server_xml:url}
rendered = $${directory_tomcat:catalina_conf}/server.xml
extensions = jinja2.ext.do
context = import json_module json
section palo_etl_parameter palo_etl_parameter
raw palo_etl_server_war ${palo_etl_download:location}
# Palo etl instance
[palo_etl_parameter] [palo_etl_parameter]
tomcat_port = 8888 tomcat_port = 8443
tomcat_host = $${:ipv6} tomcat_host = $${:ipv6}
tomcat_server_port = 8006
ipv4 = $${instance-parameter:ipv4-random} ipv4 = $${instance-parameter:ipv4-random}
ipv6 = $${instance-parameter:ipv6-random} ipv6 = $${instance-parameter:ipv6-random}
palo_etl_war = ${palo_etl_download:location} palo_etl_war = ${palo_etl_download:location}
keystore_file = $${keystore_import:keystore_file}
keystore_pass = $${keystore_import:keystore_pass}
[publish-connection-parameter] [keystore_import]
recipe = slapos.cookbook:publish recipe = plone.recipe.command
palo_olap_url = $${palo_olap_parameter:ipv4}:$${palo_olap_parameter:palo_olap_port} command =
palo_olap_ipv6_url = [$${palo_olap_parameter:ipv6}]:$${palo_olap_parameter:palo_olap_port} ${java:location}/bin/keytool \
palo_olap_admin_url = $${palo_olap_parameter:ipv4}:$${palo_olap_parameter:palo_olap_admin_port} -genkeypair \
palo_etl_url = http://[$${palo_etl_parameter:tomcat_host}]:$${palo_etl_parameter:tomcat_port}/etlserver/services/ETL-Server?wsdl -alias "tomcat" \
-keyalg RSA \
-keypass "$${:keystore_pass}" \
-dname "CN=Web Server,OU=Unit,O=Organization,L=City,S=State,C=Country" \
-keystore "$${:keystore_file}" \
-storepass "$${:keystore_pass}"
keystore_file = $${directory_palo_etl:catalina_base}/.keystore
keystore_pass = insecure
[directory_palo_etl]
<= directory
catalina_base = $${:var}/palo_etl
catalina_logs = $${:catalina_base}/logs
catalina_temp = $${:catalina_base}/temp
catalina_webapps = $${:catalina_base}/webapps
catalina_work = $${:catalina_base}/work
catalina_conf = $${:catalina_base}/conf
[palo_etl_server_xml]
recipe = slapos.recipe.template:jinja2
template = ${template_server_xml:target}
rendered = $${directory_palo_etl:catalina_conf}/server.xml
context =
section palo_etl_parameter palo_etl_parameter
raw palo_etl_server_war ${palo_etl_download:location}
[palo_etl]
recipe = slapos.cookbook:wrapper
command-line = ${tomcat:location}/bin/catalina.sh run
wrapper-path = $${directory:service}/palo_etl
environment =
JAVA_HOME = ${java:location}
CATALINA_BASE = $${directory_palo_etl:catalina_base}
dependencies = $${palo_etl_server_xml:rendered}
template-directory {{ palo_olap_repository_location }}/Api template-directory {{ palo_olap_repository_location }}/Api
http {{ palo_olap_parameter.ipv4 }} {{ palo_olap_parameter.palo_olap_port }}
http {{ palo_olap_parameter.ipv6 }} {{ palo_olap_parameter.palo_olap_port }} http {{ palo_olap_parameter.ipv6 }} {{ palo_olap_parameter.palo_olap_port }}
https {{ palo_olap_parameter.palo_olap_ssl_port }}
encryption optional
key-files {{ palo_olap_parameter['key-file'] }} {{ palo_olap_parameter['key-file'] }} {{ palo_olap_parameter['dh1024-file'] }}
extensions {{ palo_olap_parameter.extensions_dir }}
log {{ directory_log }}/palo_olap.log log {{ directory_log }}/palo_olap.log
verbose debug verbose debug
{% if erp5_url %} {% if erp5_url %}
workerlogin authorization workerlogin authorization
worker {{ erp5_login_worker_path }} {{ directory_log }}/erp5_login_worker.log {{ erp5_url }} worker {{ erp5_login_worker_path }} {{ directory_log }}/erp5_login_worker.log {{ erp5_url }}
admin {{ palo_olap_parameter.ipv4 }} {{ palo_olap_parameter.palo_olap_admin_port }} #admin {{ palo_olap_parameter.ipv4 }} {{ palo_olap_parameter.palo_olap_admin_port }}
{% else %} {% else %}
# no authorization enabled, as instance parameter erp5_url was not defined # no authorization enabled, as instance parameter erp5_url was not defined
{% endif %} {% endif %}
<?xml version='1.0' encoding='utf-8'?> <?xml version='1.0' encoding='utf-8'?>
<!--
Licensed to the Apache Software Foundation (ASF) under one or more
contributor license agreements. See the NOTICE file distributed with
this work for additional information regarding copyright ownership.
The ASF licenses this file to You under the Apache License, Version 2.0
(the "License"); you may not use this file except in compliance with
the License. You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
-->
<!-- Note: A "Server" is not itself a "Container", so you may not
define subcomponents such as "Valves" at this level.
Documentation at /docs/config/server.html
Debug: {{ repr(palo_etl_parameter) }}
-->
<Server port="-1" shutdown="SHUTDOWN"> <Server port="-1" shutdown="SHUTDOWN">
<!--APR library loader. Documentation at /docs/apr.html -->
<Listener className="org.apache.catalina.core.AprLifecycleListener" SSLEngine="on" />
<!--Initialize Jasper prior to webapps are loaded. Documentation at /docs/jasper-howto.html -->
<Listener className="org.apache.catalina.core.JasperListener" />
<!-- Prevent memory leaks due to use of particular java/javax APIs-->
<Listener className="org.apache.catalina.core.JreMemoryLeakPreventionListener" />
<!-- JMX Support for the Tomcat server. Documentation at /docs/non-existent.html -->
<!--
<Listener className="org.apache.catalina.mbeans.ServerLifecycleListener" />
<Listener className="org.apache.catalina.mbeans.GlobalResourcesLifecycleListener" />
-->
<!--
<GlobalNamingResources>
<Resource name="UserDatabase" auth="Container"
type="org.apache.catalina.UserDatabase"
description="User database that can be updated and saved"
factory="org.apache.catalina.users.MemoryUserDatabaseFactory"
pathname="conf/tomcat-users.xml" />
</GlobalNamingResources>
-->
<Service name="Catalina"> <Service name="Catalina">
<Connector address="{{ palo_etl_parameter['tomcat_host'] }}" port="{{ palo_etl_parameter['tomcat_port'] }}" <Connector
protocol="org.apache.coyote.http11.Http11Protocol"
address="{{ palo_etl_parameter['tomcat_host'] }}"
port="{{ palo_etl_parameter['tomcat_port'] }}"
maxThreads="10" maxThreads="10"
minSpareThreads="3" scheme="https"
maxSpareThreads="7" secure="true"
connectionTimeout="20000" clientAuth="false"
enableLookups="false" sslProtocol="TLS"
maxHttpHeaderSize="8192" SSLEnabled="true"
protocol="HTTP/1.1" keypass="{{ palo_etl_parameter['keystore_pass'] }}"
useBodyEncodingForURI="true" keystore="{{ palo_etl_parameter['keystore_file'] }}"
redirectPort="8443" />
acceptCount="100"
disableUploadTimeout="true" />
<Context path="/" docBase="{{ palo_etl_parameter['palo_etl_war'] }}/etlserver.war" <Context path="/" docBase="{{ palo_etl_parameter['palo_etl_war'] }}/etlserver.war"
debug="1" privileged="true"> privileged="true">
<Realm className="org.apache.catalina.realm.UserDatabaseRealm"
resourceName="UserDatabase"/>
</Context> </Context>
<Engine name="Catalina" defaultHost="localhost"> <Engine name="Catalina" defaultHost="localhost">
<Realm className="org.apache.catalina.realm.UserDatabaseRealm" <Host name="localhost" appBase="webapps"
resourceName="UserDatabase"/>
<Host name="localhost" appBase="webapps" debug="1"
unpackWARs="true" autoDeploy="true" unpackWARs="true" autoDeploy="true"
xmlValidation="false" xmlNamespaceAware="false"> xmlValidation="false" xmlNamespaceAware="false">
</Host> </Host>
</Engine> </Engine>
</Service> </Service>
......
...@@ -6,17 +6,17 @@ extends = ...@@ -6,17 +6,17 @@ extends =
parts = parts =
palo_olap palo_olap
palo_etl palo_etl_download
template_server_xml template_server_xml
template_palo_ini
slapos-cookbook slapos-cookbook
instance-profile instance-profile
[instance-profile] [instance-profile]
recipe = slapos.recipe.template recipe = slapos.recipe.template
url = ${:_profile_base_location_}/instance.cfg.in url = ${:_profile_base_location_}/instance.cfg.in
output = ${buildout:directory}/instance.cfg output = ${buildout:directory}/instance.cfg
#md5sum = #md5sum = b30db17333d11ce11f486a96e68ad9b1
mode = 0644 mode = 0644
[palo_etl_download] [palo_etl_download]
...@@ -32,11 +32,9 @@ mode = 644 ...@@ -32,11 +32,9 @@ mode = 644
[template_server_xml] [template_server_xml]
< = download-base < = download-base
filename = server.xml.in filename = server.xml.in
# md5sum = #md5sum = 19daa17468de0940e0d10d4bd4adf86e
[palo_etl]
recipe = plone.recipe.command
command = echo "etl downlaoded in ${palo_etl_download:location}"
needs = ${palo_etl_download:url}
[versions] [template_palo_ini]
< = download-base
filename = palo.ini.in
#md5sum = d85f449ce322eba4d75b6ad28c8bece8
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment