Skip to content

slapos
slapos
Commit 82d2d645 authored by Kazuhiko Shiozaki's avatar Kazuhiko Shiozaki
Browse files
Options

Merge remote-tracking branch 'origin/master' into erp5-component

parents 252a5425 5b2b751e
Show whitespace changes
Inline Side-by-side
Showing with 406 additions and 240 deletions
component/squid/buildout.cfg
View file @ 82d2d645
......@@ -9,8 +9,8 @@ extends =
[squid]
recipe = hexagonit.recipe.cmmi
url = http://www.squid-cache.org/Versions/v3/3.2/squid-3.2.1.tar.gz
md5sum = 3fb81acc6b70a432e3f0d8a0491056dc
url = http://www1.at.squid-cache.org/Versions/v3/3.4/squid-3.4.4.tar.gz
md5sum = b36ab3696485a86b43d9194ac9b6b7fe
configure-options =
--disable-dependency-tracking
--disable-translation
......
software/apache-frontend/common.cfg
View file @ 82d2d645
......@@ -4,6 +4,7 @@ extends =
../../component/git/buildout.cfg
../../stack/slapos.cfg
../../component/dash/buildout.cfg
../../component/binutils/buildout.cfg
../../component/lxml-python/buildout.cfg
../../component/apache/buildout.cfg
......@@ -13,10 +14,13 @@ extends =
../../component/logrotate/buildout.cfg
../../component/rdiff-backup/buildout.cfg
../../component/squid/buildout.cfg
# Monitoring stack
../../stack/monitor/buildout.cfg
parts +=
slapos-cookbook
slapos-toolbox
eggs
template
template-apache-frontend
template-apache-replicate
......@@ -36,12 +40,16 @@ recipe = zc.recipe.egg
eggs =
${lxml-python:egg}
slapos.toolbox
plone.recipe.command
scripts =
killpidfromfile
onetimedownload
[eggs]
recipe = zc.recipe.egg
eggs =
plone.recipe.command
[check-recipe]
recipe = plone.recipe.command
stop-on-error = true
......@@ -59,20 +67,20 @@ mode = 0644
[template-apache-frontend]
recipe = slapos.recipe.template
url = ${:_profile_base_location_}/instance-apache-frontend.cfg
md5sum = f0a507fed2b1dcab5530c892adce8327
md5sum = f5ec3d3b29d20ccdb00e3b64aa588fa5
output = ${buildout:directory}/template-apache-frontend.cfg
mode = 0644
[template-apache-replicate]
recipe = slapos.recipe.build:download
url = ${:_profile_base_location_}/instance-apache-replicate.cfg.in
md5sum = 02658d39fa429fef7faba9658e4f1474
md5sum = 82c88a4b4856bfffec3d7ef24e372f38
mode = 0644
[template-slave-list]
recipe = slapos.recipe.build:download
url = ${:_profile_base_location_}/templates/apache-custom-slave-list.cfg.in
md5sum = f002a8fc8fc5d18adbd8ac1ee054e852
md5sum = 5c49a66a0fe4b92ab21ef7148ea4e5ff
mode = 640
[template-slave-configuration]
......@@ -90,7 +98,7 @@ mode = 640
[template-apache-frontend-configuration]
recipe = slapos.recipe.build:download
url = ${:_profile_base_location_}/templates/apache.conf.in
md5sum = bae89ebc6c5e75b12535fbd6c37f647d
md5sum = 72922908c1f4e72c92bb03e072660c7c
mode = 640
[template-apache-cached-configuration]
......@@ -121,7 +129,7 @@ mode = 640
[template-default-virtualhost]
recipe = slapos.recipe.build:download
url = ${:_profile_base_location_}/templates/000.conf.in
md5sum = c2bbf029e6adc432de0884fb5cf5d2ab
md5sum = b40ffdab93a80b40046e3bbb2f7a58bc
mode = 640
[template-default-slave-virtualhost]
......@@ -139,7 +147,7 @@ mode = 640
[template-squid-configuration]
recipe = slapos.recipe.build:download
url = ${:_profile_base_location_}/templates/squid.conf.jinja2
md5sum = 9f70474181372d34c8cd203f24ab546e
md5sum = f17753fa87da074bc949b2967a330099
mode = 640
[template-empty]
......@@ -147,3 +155,10 @@ recipe = slapos.recipe.build:download
url = ${:_profile_base_location_}/templates/empty.in
md5sum = c2314c3a9c3412a38d14b312d3df83c1
mode = 640
[template-wrapper]
recipe = slapos.recipe.template
url = ${:_profile_base_location_}/templates/wrapper.in
output = ${buildout:directory}/template-wrapper.cfg
mode = 0644
md5sum = 8cde04bfd0c0e9bd56744b988275cfd8
software/apache-frontend/instance-apache-frontend.cfg
View file @ 82d2d645
......@@ -21,11 +21,33 @@ parts =
promise-squid
dynamic-template-default-vh
not-found-html
promise-frontend-apache-configuration
promise-cached-apache-configuration
promise-apache-frontend-v4-https
promise-apache-frontend-v4-http
promise-apache-frontend-v6-https
promise-apache-frontend-v6-http
promise-apache-cached
## Monitoring part
###Parts to add for monitoring
certificate-authority
cron-entry-monitor
cron-entry-rss
deploy-index
deploy-settings-cgi
deploy-status-cgi
setup-static-files
certificate-authority
zero-parameters
public-symlink
cgi-httpd-wrapper
cgi-httpd-graceful-wrapper
monitor-promise
monitor-instance-log-access
## Monitor for apache
monitor-current-log-access
monitor-backup-log-access
extends = ${monitor-template:output}
eggs-directory = ${buildout:eggs-directory}
......@@ -95,6 +117,14 @@ apache-directory = ${apache-2.2:location}
apache-ipv6 = $${instance-parameter:ipv6-random}
apache-https-port = $${instance-parameter:configuration.port}
[monitor-current-log-access]
< = monitor-directory-access
source = $${directory:log}
[monitor-backup-log-access]
< = monitor-directory-access
source = $${directory:logrotate-backup}
[jinja2-template-base]
recipe = slapos.recipe.template:jinja2
rendered = $${buildout:directory}/$${:filename}
......@@ -140,6 +170,7 @@ extra-context =
section logrotate_dict logrotate
section frontend_configuration frontend-configuration
section apache_configuration apache-configuration
section connection_information_dict publish-connection-informations
[dynamic-custom-group-template-slave-list]
< = jinja2-template-base
......@@ -292,6 +323,10 @@ pid-file = $${directory:run}/httpd.pid
protected-path = /
access-control-string = none
cached-rewrite-file = $${directory:etc}/apache_rewrite_cached.txt
frontend-configuration-verification = ${apache-2.2:location}/bin/httpd -Sf $${:frontend-configuration}
frontend-graceful-command = $${:frontend-configuration-verification}; if [ $? -eq 0 ]; then kill -USR1 $(cat $${:pid-file}); fi
cached-configuration-verification = ${apache-2.2:location}/bin/httpd -Sf $${:cached-configuration}
cached-graceful-command = $${:cached-configuration-verification}; if [ $? -eq 0 ]; then kill -USR1 $(cat $${apache-configuration:cache-pid-file}); fi
# Apache for cache configuration
cache-access-log = $${directory:log}/frontend-apache-access-cached.log
......@@ -381,7 +416,7 @@ name = apache
log = $${apache-configuration:error-log} $${apache-configuration:access-log}
frequency = daily
rotatep-num = 30
post = ${buildout:bin-directory}/killpidfromfile $${apache-configuration:pid-file} SIGUSR1
post = $${apache-configuration:frontend-graceful-command}
sharedscripts = true
notifempty = true
create = true
......@@ -393,7 +428,7 @@ name = apache-cached
log = $${apache-configuration:cache-error-log} $${apache-configuration:cache-access-log}
frequency = daily
rotatep-num = 30
post = ${buildout:bin-directory}/killpidfromfile $${apache-configuration:cache-pid-file} SIGUSR1
post = $${apache-configuration:cached-graceful-command}
sharedscripts = true
notifempty = true
create = true
......@@ -470,15 +505,39 @@ port = $${apache-configuration:cache-port}
# End of Squid part
### Apaches Graceful and promises
[frontend-apache-graceful]
recipe = slapos.cookbook:wrapper
command-line = ${buildout:bin-directory}/killpidfromfile $${apache-configuration:pid-file} SIGUSR1
wrapper-path = $${directory:etc-run}/frontend-apache-graceful
< = jinja2-template-base
template = ${template-wrapper:output}
rendered = $${directory:etc-run}/frontend-apache-safe-graceful
mode = 0700
extra-context =
key content apache-configuration:frontend-graceful-command
[cached-apache-graceful]
recipe = slapos.cookbook:wrapper
command-line = ${buildout:bin-directory}/killpidfromfile $${apache-configuration:cache-pid-file} SIGUSR1
wrapper-path = $${directory:etc-run}/cached-apache-graceful
< = jinja2-template-base
template = ${template-wrapper:output}
rendered = $${directory:etc-run}/cached-apache-safe-graceful
mode = 0700
extra-context =
key content apache-configuration:cached-graceful-command
# Promises checking configuration:
[promise-frontend-apache-configuration]
< = jinja2-template-base
template = ${template-wrapper:output}
rendered = $${directory:promise}/frontend-apache-configuration-promise
mode = 0700
extra-context =
key content apache-configuration:frontend-configuration-verification
[promise-cached-apache-configuration]
< = jinja2-template-base
template = ${template-wrapper:output}
rendered = $${directory:promise}/cached-apache-configuration-promise
mode = 0700
extra-context =
key content apache-configuration:cached-configuration-verification
[promise-apache-frontend-v4-https]
recipe = slapos.cookbook:check_port_listening
......
software/apache-frontend/instance-apache-replicate.cfg.in
View file @ 82d2d645
......@@ -59,20 +59,24 @@ sla-{{ parameter }} = {{ slapparameter_dict.pop( sla_key + parameter ) }}
[replicate]
<= slap-connection
recipe = slapos.cookbook:request
recipe = slapos.cookbook:requestoptional
software-url = ${slap-connection:software-release-url}
software-type = {{frontend_type}}
return = private-ipv4 public-ipv4 slave-instance-information-list
return = private-ipv4 public-ipv4 slave-instance-information-list monitor_url
config = {{ ' '.join(slapparameter_dict.keys()) + ' ' + slave_list_name }}
{% for parameter, value in slapparameter_dict.iteritems() -%}
config-{{parameter}} = {{ value }}
{% endfor -%}
config-{{ slave_list_name }} = {{ json_module.dumps(slave_instance_list) }}
connection-monitor_url =
[publish-information]
recipe = slapos.cookbook:publish
domain = {{ slapparameter_dict.get('domain') }}
slave-amount = {{ slave_instance_list | length }}
{% for frontend in frontend_section_list %}
{{ frontend }}-monitor-url = {{ '${' + frontend + ':connection-monitor_url}' }}
{% endfor -%}
{% for frontend in frontend_list -%}
#{{frontend}}-private-ipv4 = ${request-{{frontend}}:private-ipv4}
{% endfor -%}
......
software/apache-frontend/software.cfg
View file @ 82d2d645
......@@ -2,120 +2,106 @@
extends = common.cfg
[versions]
Jinja2 = 2.6
Werkzeug = 0.8.3
apache-libcloud = 0.12.3
[versions]
Jinja2 = 2.7.2
MarkupSafe = 0.23
PyRSS2Gen = 1.1
Werkzeug = 0.9.4
apache-libcloud = 0.14.1
async = 0.6.1
buildout-versions = 1.7
gitdb = 0.5.4
hexagonit.recipe.cmmi = 2.0
meld3 = 0.6.10
pycrypto = 2.6
rdiff-backup = 1.0.5
slapos.recipe.build = 0.11.6
slapos.recipe.cmmi = 0.1.1
slapos.recipe.template = 2.4.2
slapos.toolbox = 0.34.0
smmap = 0.8.2
z3c.recipe.scripts = 1.0.1
cliff = 1.4.4
cmd2 = 0.6.5.1
prettytable = 0.7.2
requests = 1.2.3
slapos.cookbook = 0.82
cffi = 0.8.2
cmd2 = 0.6.7
cns.recipe.symlink = 0.2.3
collective.recipe.template = 1.11
cryptography = 0.4
gitdb = 0.5.4
hexagonit.recipe.cmmi = 2.0
inotifyx = 0.2.0-1
itsdangerous = 0.24
lxml = 3.3.5
meld3 = 1.0.0
mr.developer = 1.30
netaddr = 0.7.11
plone.recipe.command = 1.1
pyOpenSSL = 0.14
pycrypto = 2.6.1
pyparsing = 2.0.2
pytz = 2014.3
rdiff-backup = 1.0.5
six = 1.6.1
slapos.cookbook = 0.87
slapos.recipe.build = 0.12
slapos.recipe.cmmi = 0.2
slapos.recipe.template = 2.5
slapos.toolbox = 0.39.2
smmap = 0.8.2
stevedore = 0.15
# Required by:
# cffi==0.8.2
pycparser = 2.10
# Required by:
# slapos.cookbook==0.82
lock-file = 2.0
# Required by:
# slapos.core==0.35.1
# slapos.toolbox==0.34.0
Flask = 0.9
# slapos.core==1.0.5
# slapos.toolbox==0.39.2
Flask = 0.10.1
# Required by:
# slapos.toolbox==0.34.0
# slapos.toolbox==0.39.2
GitPython = 0.3.2.RC1
# Required by:
# slapos.toolbox==0.34.0
atomize = 0.1.1
# Required by:
# slapos.toolbox==0.34.0
feedparser = 5.1.3
# Required by:
# slapos.cookbook==0.82
inotifyx = 0.2.0
# slapos.toolbox==0.39.2
atomize = 0.2.0
# Required by:
# slapos.cookbook==0.82
# slapos.core==0.35.1
# xml-marshaller==0.9.7
lxml = 3.1.2
# slapos.core==1.0.5
cliff = 1.6.1
# Required by:
# slapos.cookbook==0.82
netaddr = 0.7.10
# paramiko==1.14.0
ecdsa = 0.11
# Required by:
# slapos.core==0.35.1
netifaces = 0.10.3
# slapos.toolbox==0.39.2
feedparser = 5.1.3
# Required by:
# slapos.toolbox==0.34.0
paramiko = 1.10.1
# slapos.cookbook==0.87
lock-file = 2.0
# Required by:
# slapos.toolbox==0.34.0
psutil = 0.7.0
# slapos.core==1.0.5
netifaces = 0.10.4
# Required by:
# slapos.core==0.35.1
pyflakes = 0.7
# slapos.toolbox==0.39.2
paramiko = 1.14.0
# Required by:
# slapos.cookbook==0.82
pytz = 2013b
# slapos.toolbox==0.39.2
psutil = 2.1.1
# Required by:
# slapos.cookbook==0.82
# slapos.core==0.35.1
# slapos.toolbox==0.34.0
# zc.buildout==1.6.0-dev-SlapOS-010
# zc.recipe.egg==1.3.2
setuptools = 0.6c12dev-r88846
# cffi==0.8.2
pycparser = 2.10
# Required by:
# slapos.cookbook==0.82
# slapos.toolbox==0.34.0
slapos.core = 0.35.1
# slapos.core==1.0.5
requests = 2.3.0
# Required by:
# slapos.core==0.35.1
supervisor = 3.0b1
# slapos.toolbox==0.39.2
slapos.core = 1.0.5
# Required by:
# slapos.core==0.35.1
unittest2 = 0.5.1
# slapos.core==1.0.5
supervisor = 3.0
# Required by:
# slapos.cookbook==0.82
# slapos.toolbox==0.34.0
# slapos.toolbox==0.39.2
xml-marshaller = 0.9.7
# Required by:
# slapos.core==0.35.1
zope.interface = 4.0.5
# slapos.core==1.0.5
zope.interface = 4.1.1
[networkcache]
# signature certificates of the following uploaders.
......
software/apache-frontend/templates/000.conf.in
View file @ 82d2d645
......@@ -2,9 +2,7 @@
ServerName www.example.org
SSLEngine on
SSLProxyEngine on
SSLProtocol -ALL +SSLv3 +TLSv1
SSLHonorCipherOrder On
SSLCipherSuite RC4-SHA:HIGH:!ADH
SSLProtocol ALL -SSLv2
# Rewrite part
ProxyVia On
......
software/apache-frontend/templates/apache-custom-slave-list.cfg.in
View file @ 82d2d645
......@@ -53,7 +53,7 @@ log = {{slave_parameter_dict.get('access_log')}} {{slave_parameter_dict.get('err
backup = {{ '${' + slave_directory_section + ':log-folder}' }}
frequency = daily
rotatep-num = 30
post = ${buildout:bin-directory}/killpidfromfile {{ apache_configuration.get('pid-file') }} SIGUSR1
post = {{ apache_configuration.get('frontend-graceful-command') }}
sharedscripts = true
notifempty = true
create = true
......@@ -172,6 +172,10 @@ private-ipv4 = {{ local_ipv4 }}
{% if extra_slave_instance_list -%}
slave-instance-information-list = {{ json_module.dumps(slave_instance_information_list) }}
{% endif -%}
{% do connection_information_dict.pop('recipe') %}
{% for key, value in connection_information_dict.iteritems() -%}
{{ key }} = {{ value }}
{% endfor %}
{% do part_list.append('cached-rewrite-rules') -%}
[cached-rewrite-rules]
......
software/apache-frontend/templates/apache.conf.in
View file @ 82d2d645
......@@ -120,9 +120,7 @@ SSLSessionCache shmcb:/{{ httpd_mod_ssl_cache_directory }}/ssl_scache(512000)
SSLSessionCacheTimeout 300
SSLRandomSeed startup /dev/urandom 256
SSLRandomSeed connect builtin
SSLProtocol -ALL +SSLv3 +TLSv1
SSLHonorCipherOrder On
SSLCipherSuite RC4-SHA:HIGH:!ADH
SSLProtocol ALL -SSLv2
<FilesMatch "\.(cgi|shtml|phtml|php)$">
SSLOptions +StdEnvVars
</FilesMatch>
......
software/apache-frontend/templates/squid.conf.jinja2
View file @ 82d2d645
......@@ -23,6 +23,8 @@ header_replace X-Forwarded-For
follow_x_forwarded_for allow all
forwarded_for on
cache_dir aufs {{ cache_path }} 5000 16 256
# Use 1Go of RAM
cache_mem 1024 MB
# But do not keep big object in RAM
......
software/apache-frontend/templates/wrapper.in 0 → 100644
View file @ 82d2d645
#!${dash-output:dash}
{{ content }}
\ No newline at end of file
stack/monitor/buildout.cfg
View file @ 82d2d645
......@@ -6,28 +6,22 @@ extends =
../../component/dash/buildout.cfg
../../component/dcron/buildout.cfg
../../component/openssl/buildout.cfg
../../component/nginx/buildout.cfg
parts +=
backup-script-template
collective.recipe.template-egg
parts =
slapos-cookbook
dcron
download-static-files
eggs
monitor-eggs
extra-eggs
make-rss
monitor-bin
monitor-template
nginx
rss-bin
slapos-cookbook
static-folder
template-nginx-conf
[collective.recipe.template-egg]
[monitor-eggs]
recipe = zc.recipe.egg
eggs = collective.recipe.template
PyRSS2Gen
eggs =
collective.recipe.template
cns.recipe.symlink
[extra-eggs]
recipe = zc.recipe.egg
......@@ -39,8 +33,8 @@ eggs =
[make-rss-script]
recipe = slapos.recipe.template
url = ${:_profile_base_location_}/make-rss.sh.in
md5sum = 8d3a4b212784b591316b8b93d6bd163e
output = ${buildout:directory}/make-rss.sh.in
md5sum = 98c8f6fd81e405b0ad10db07c3776321
output = ${buildout:directory}/template-make-rss.sh.in
mode = 0644
[monitor-template]
......@@ -48,14 +42,15 @@ recipe = slapos.recipe.template
url = ${:_profile_base_location_}/monitor.cfg.in
output = ${buildout:directory}/monitor.cfg
filename = monitor.cfg
md5sum = 4db2be5e8cabb5dd1d23743e0db5b02a
md5sum = bd592a0f0c41ec15c643c4e91e9ec5cc
mode = 0644
[monitor-bin]
recipe = hexagonit.recipe.download
url = ${:_profile_base_location_}/${:filename}
download-only = true
md5sum = 05bbb70f6f69dc4f3fa83dc0f8c9960e
md5sum = 1e7b4698f6627150b1eb783b06f8b13a
destination = ${buildout:directory}/parts/monitor-template-monitor-bin
filename = monitor.py.in
mode = 0644
......@@ -64,6 +59,7 @@ recipe = hexagonit.recipe.download
url = ${:_profile_base_location_}/webfiles/${:filename}
download-only = true
md5sum = 91ac749f86aecc0c383d93e51e15a572
destination = ${buildout:directory}/parts/monitor-index
filename = index.cgi.in
mode = 0644
......@@ -71,7 +67,8 @@ mode = 0644
recipe = hexagonit.recipe.download
url = ${:_profile_base_location_}/webfiles/${:filename}
download-only = true
md5sum = aa375a4225e2587b22f68c28cafd7871
destination = ${buildout:directory}/parts/monitor-template-index
md5sum = e0d2aaeffc046b2ac6d9d717e1ba321d
filename = index.html.jinja2
mode = 0644
......@@ -79,7 +76,8 @@ mode = 0644
recipe = hexagonit.recipe.download
url = ${:_profile_base_location_}/webfiles/${:filename}
download-only = true
md5sum = 4e5b7fc5c5c237836c7c81fe3e3bb903
md5sum = aa2764cab87e457410435974f729e906
destination = ${buildout:directory}/parts/monitor-template-status-cgi
filename = status.cgi.in
mode = 0644
......@@ -88,6 +86,7 @@ recipe = hexagonit.recipe.download
url = ${:_profile_base_location_}/webfiles/${:filename}
download-only = true
md5sum = 18574b804da0c65d8670959f9e7c4774
destination = ${buildout:directory}/parts/monitor-template-settings-cgi
filename = settings.cgi.in
mode = 0644
......@@ -95,7 +94,8 @@ mode = 0644
recipe = hexagonit.recipe.download
url = ${:_profile_base_location_}/${:filename}
download-only = true
md5sum = 427d5135ca475f2a4c4e69a0da6ed8c4
md5sum = 5f1b93ccdea7c3031aef396154c64938
destination = ${buildout:directory}/parts/monitor-template-rss-bin
filename = status2rss.py
mode = 0644
......@@ -108,18 +108,9 @@ logfile = $${directory:log}/crond.log
[download-static-files]
recipe = hexagonit.recipe.download
url = https://github.com/Sebatyne/staticForMonitoring/blob/3f0e93cec706c7ad311ddbf1ebf996965ce2f0a3/static-files.tar.gz?raw=true
url = https://github.com/SlapOS/staticForMonitoring/blob/db670e7568871c69a64916d462ccb57629f1c77d/static-files.tar.gz?raw=true
download-only = true
md5sum = e98585b85634de48240b2c215e946769
md5sum = 9e3feb2b520620d5b8d478eb9a9be6de
filename = static-files.tar.gz
destination = ${buildout:directory}/parts/monitor-static-files
mode = 0644
[eggs]
recipe = z3c.recipe.scripts
eggs =
slapos.cookbook
PyRSS2Gen
[versions]
PyRSS2Gen = 1.1
Jinja2 = 2.6
stack/monitor/make-rss.sh.in
View file @ 82d2d645
#!${dash-output:dash}
STATUS=$${monitor-parameters:result-dir}
RSS_FILE=$${monitor-parameters:rss-path}
STATUS_DB={{ monitor_parameters['db-path'] }}
RSS_FILE={{ monitor_parameters['rss-path'] }}
PYTHON=${buildout:directory}/bin/${extra-eggs:interpreter}
STATUS2RSS=${rss-bin:location}/${rss-bin:filename}
NAME=`basename $STATUS`
cat $STATUS/* | $PYTHON $STATUS2RSS "Monitoring RSS feed" "https://[$${slap-parameters:ipv6-random}]:$${monitor-parameters:port}/$${deploy-index:filename}" > $RSS_FILE
$PYTHON $STATUS2RSS "Monitoring RSS feed" "{{ monitor_parameters['url'] }}/{{ monitor_parameters['index-filename'] }}" $STATUS_DB > $RSS_FILE
stack/monitor/monitor.cfg.in
View file @ 82d2d645
......@@ -7,104 +7,108 @@ key = $${slap-connection:key-file}
cert = $${slap-connection:cert-file}
[monitor-parameters]
monitor-dir = $${directory:var}/monitor
result-dir = $${:monitor-dir}/bool
json-filename = monitor.json
json-path = $${:monitor-dir}/$${:json-filename}
rss-path = $${:public-cgi}/$${:rss-filename}
json-path = $${monitor-directory:monitor-result}/$${:json-filename}
rss-filename = rssfeed.html
executable = $${directory:bin}/monitor.py
cgi-bin = $${directory:cgi-bin}
monitoring-cgi = $${directory:monitoring-cgi}
knowledge0-cgi = $${directory:knowledge0-cgi}
public-cgi = $${directory:public-cgi}
rss-path = $${monitor-directory:public-cgi}/$${:rss-filename}
executable = $${monitor-directory:bin}/monitor.py
port = 9685
htaccess-file = $${monitor-directory:etc}/.htaccess-monitor
url = https://[$${slap-parameters:ipv6-random}]:$${:port}
index-filename = index.cgi
index-path = $${monitor-directory:www}/$${:index-filename}
db-path = $${monitor-directory:etc}/monitor.db
[directory]
[monitor-directory]
recipe = slapos.cookbook:mkdirectory
# Standard directory needed by monitoring stack
home = $${buildout:directory}
etc = $${:home}/etc
bin = $${:home}/bin
srv = $${:home}/srv
var = $${:home}/var
log = $${:var}/log
run = $${:var}/run
service = $${:etc}/service/
etc-run = $${:etc}/run/
tmp = $${:home}/tmp
promise = $${:etc}/promise
cron-entries = $${:etc}/cron.d
crontabs = $${:etc}/crontabs
cronstamps = $${:etc}/cronstamps
promises = $${:etc}/promise
ca-dir = $${:srv}/ssl
www = $${:var}/www
cgi-bin = $${:var}/cgi-bin
monitoring-cgi = $${:cgi-bin}/monitoring
knowledge0-cgi = $${:cgi-bin}/zero-knowledge
cron-entries = $${:etc}/cron.d
crontabs = $${:etc}/crontabs
cronstamps = $${:etc}/cronstamps
log = $${:var}/log
monitor = $${:etc}/monitor
monitor-result = $${monitor-parameters:monitor-dir}
monitor-result-bool = $${monitor-parameters:result-dir}
promise = $${:etc}/promise
public-cgi = $${:cgi-bin}/public
run = $${:var}/run
service = $${:etc}/service/
tmp = $${:home}/tmp
www = $${:var}/www
monitor-custom-scripts = $${:etc}/monitor
monitor-result = $${:var}/monitor
private-directory = $${:srv}/monitor-private
[public-symlink]
recipe = cns.recipe.symlink
symlink = $${monitor-parameters:public-cgi} = $${directory:www}/public
symlink = $${monitor-directory:public-cgi} = $${monitor-directory:www}/public
autocreate = true
[cron]
recipe = slapos.cookbook:cron
dcrond-binary = ${dcron:location}/sbin/crond
cron-entries = $${directory:cron-entries}
crontabs = $${directory:crontabs}
cronstamps = $${directory:cronstamps}
cron-entries = $${monitor-directory:cron-entries}
crontabs = $${monitor-directory:crontabs}
cronstamps = $${monitor-directory:cronstamps}
catcher = $${cron-simplelogger:wrapper}
binary = $${directory:service}/crond
binary = $${monitor-directory:service}/crond
# Add log to cron
[cron-simplelogger]
recipe = slapos.cookbook:simplelogger
wrapper = $${directory:bin}/cron_simplelogger
log = $${directory:log}/cron.log
wrapper = $${monitor-directory:bin}/cron_simplelogger
log = $${monitor-directory:log}/cron.log
[cron-entry-monitor]
<= cron
recipe = slapos.cookbook:cron.d
name = launch-monitor
frequency = */5 * * * *
command = $${monitor-parameters:executable} -a
command = $${deploy-monitor-script:rendered} -a
[cron-entry-rss]
<= cron
recipe = slapos.cookbook:cron.d
name = build-rss
frequency = */5 * * * *
command = $${make-rss:output}
command = $${make-rss:rendered}
[setup-static-files]
recipe = hexagonit.recipe.download
url = ${download-static-files:destination}/${download-static-files:filename}
filename = static
destination = $${directory:www}
destination = $${monitor-directory:www}
ignore-existing = true
mode = 0644
[deploy-index]
recipe = slapos.recipe.template:jinja2
template = ${index:location}/${index:filename}
rendered = $${directory:www}/$${:filename}
filename = index.cgi
rendered = $${monitor-parameters:index-path}
mode = 0744
context =
key cgi_directory monitor-parameters:cgi-bin
key cgi_directory monitor-directory:cgi-bin
raw index_template $${deploy-index-template:location}/$${deploy-index-template:filename}
key password zero-parameters:monitor-password
raw extra_eggs_interpreter ${buildout:directory}/bin/${extra-eggs:interpreter}
raw default_page /index.cgi?script=$${monitor-parameters:knowledge0-cgi}%2F$${deploy-settings-cgi:filename}
raw default_page /welcome.html
[deploy-index-template]
recipe = hexagonit.recipe.download
url = ${index-template:location}/$${:filename}
destination = $${directory:www}
destination = $${monitor-directory:www}
filename = ${index-template:filename}
download-only = true
mode = 0644
......@@ -112,24 +116,27 @@ mode = 0644
[deploy-status-cgi]
recipe = slapos.recipe.template:jinja2
template = ${status-cgi:location}/${status-cgi:filename}
rendered = $${monitor-parameters:monitoring-cgi}/$${:filename}
rendered = $${monitor-directory:monitoring-cgi}/$${:filename}
filename = status.cgi
mode = 0744
context =
key json_file monitor-parameters:json-path
key monitor_bin monitor-parameters:executable
key pwd monitor-directory:monitoring-cgi
key this_file :filename
raw python_executable ${buildout:executable}
[deploy-settings-cgi]
recipe = slapos.recipe.template:jinja2
template = ${settings-cgi:location}/${settings-cgi:filename}
rendered = $${monitor-parameters:knowledge0-cgi}/$${:filename}
rendered = $${monitor-directory:knowledge0-cgi}/$${:filename}
filename = settings.cgi
mode = 0744
context =
raw config_cfg $${buildout:directory}/knowledge0.cfg
raw timestamp $${buildout:directory}/.timestamp
raw python_executable ${buildout:executable}
key pwd monitor-parameters:knowledge0-cgi
key pwd monitor-directory:knowledge0-cgi
key this_file :filename
[deploy-monitor-script]
......@@ -138,39 +145,51 @@ template = ${monitor-bin:location}/${monitor-bin:filename}
rendered = $${monitor-parameters:executable}
mode = 0744
context =
section directory directory
section directory monitor-directory
section monitor_parameter monitor-parameters
key monitoring_file_json monitor-parameters:json-path
key monitoring_folder_bool monitor-parameters:result-dir
raw python_executable ${buildout:executable}
[deploy-rss-script]
recipe = hexagonit.recipe.download
url = ${rss-bin:destination}/${rss-bin:filename}
destination = $${directory:bin}
filename = ${rss-bin:filename}
mode = 0744
download-only = true
[make-rss]
recipe = slapos.recipe.template
url = ${make-rss-script:output}
output = $${directory:bin}/make-rss.sh
recipe = slapos.recipe.template:jinja2
template = ${make-rss-script:output}
rendered = $${monitor-directory:bin}/make-rss.sh
mode = 0744
context =
section directory monitor-directory
section monitor_parameters monitor-parameters
[monitor-htaccess]
recipe = plone.recipe.command
stop-on-error = true
htaccess-path = $${monitor-parameters:htaccess-file}
command = ${apache:location}/bin/htpasswd -cb $${:htaccess-path} admin $${zero-parameters:monitor-password}
[monitor-directory-access]
recipe = plone.recipe.command
command = ln -s $${:source} $${monitor-directory:private-directory}
source =
[monitor-instance-log-access]
recipe = plone.recipe.command
command = if [ -d $${:source} ]; then ln -s $${:source} $${monitor-directory:private-directory}/instance-logs; fi
update-command = if [ -d $${:source} ]; then ln -s $${:source} $${monitor-directory:private-directory}/instance-logs; fi
source = $${monitor-directory:home}/.slapgrid/log/
[cadirectory]
recipe = slapos.cookbook:mkdirectory
requests = $${directory:ca-dir}/requests/
private = $${directory:ca-dir}/private/
certs = $${directory:ca-dir}/certs/
newcerts = $${directory:ca-dir}/newcerts/
crl = $${directory:ca-dir}/crl/
requests = $${monitor-directory:ca-dir}/requests/
private = $${monitor-directory:ca-dir}/private/
certs = $${monitor-directory:ca-dir}/certs/
newcerts = $${monitor-directory:ca-dir}/newcerts/
crl = $${monitor-directory:ca-dir}/crl/
[certificate-authority]
recipe = slapos.cookbook:certificate_authority
openssl-binary = ${openssl:location}/bin/openssl
ca-dir = $${directory:ca-dir}
ca-dir = $${monitor-directory:ca-dir}
requests-directory = $${cadirectory:requests}
wrapper = $${directory:service}/certificate_authority
wrapper = $${monitor-directory:service}/certificate_authority
ca-private = $${cadirectory:private}
ca-certs = $${cadirectory:certs}
ca-newcerts = $${cadirectory:newcerts}
......@@ -181,8 +200,8 @@ ca-crl = $${cadirectory:crl}
recipe = slapos.cookbook:certificate_authority.request
key-file = $${cadirectory:certs}/httpd.key
cert-file = $${cadirectory:certs}/httpd.crt
executable = $${directory:bin}/cgi-httpd
wrapper = $${directory:service}/cgi-httpd
executable = $${monitor-directory:bin}/cgi-httpd
wrapper = $${monitor-directory:service}/cgi-httpd
# Put domain name
name = example.com
......@@ -214,11 +233,18 @@ input = inline:
LoadModule unixd_module modules/mod_unixd.so
LoadModule access_compat_module modules/mod_access_compat.so
LoadModule authz_core_module modules/mod_authz_core.so
LoadModule authn_core_module modules/mod_authn_core.so
LoadModule authz_host_module modules/mod_authz_host.so
LoadModule mime_module modules/mod_mime.so
LoadModule cgid_module modules/mod_cgid.so
LoadModule dir_module modules/mod_dir.so
LoadModule ssl_module modules/mod_ssl.so
LoadModule alias_module modules/mod_alias.so
LoadModule autoindex_module modules/mod_autoindex.so
LoadModule auth_basic_module modules/mod_auth_basic.so
LoadModule authz_user_module modules/mod_authz_user.so
LoadModule authn_file_module modules/mod_authn_file.so
# SSL Configuration
<IfDefine !SSLConfigured>
Define SSLConfigured
......@@ -241,16 +267,31 @@ input = inline:
# XXX: security????
Options +ExecCGI
AddHandler cgi-script .cgi
DirectoryIndex $${deploy-index:filename}
DirectoryIndex $${monitor-parameters:index-filename}
</Directory>
Alias /private/ $${monitor-directory:private-directory}/
<Directory $${monitor-directory:private-directory}>
Order Deny,Allow
Deny from env=AUTHREQUIRED
<Files ".??*">
Order Allow,Deny
Deny from all
</Files>
AuthType Basic
AuthName "Private access"
AuthUserFile "$${monitor-htaccess:htaccess-path}"
Require valid-user
Options Indexes FollowSymLinks
Satisfy all
</Directory>
output = $${directory:etc}/cgi-httpd.conf
output = $${monitor-directory:etc}/cgi-httpd.conf
listening-ip = $${slap-parameters:ipv6-random}
# XXX: randomize-me
htdocs = $${directory:www}
pid-file = $${directory:run}/cgi-httpd.pid
cgid-pid-file = $${directory:run}/cgi-httpd-cgid.pid
document-root = $${directory:www}
error-log = $${directory:log}/cgi-httpd-error-log
htdocs = $${monitor-directory:www}
pid-file = $${monitor-directory:run}/cgi-httpd.pid
cgid-pid-file = $${monitor-directory:run}/cgi-httpd-cgid.pid
document-root = $${monitor-directory:www}
error-log = $${monitor-directory:log}/cgi-httpd-error-log
[cgi-httpd-wrapper]
recipe = slapos.cookbook:wrapper
......@@ -258,15 +299,20 @@ apache-executable = ${apache:location}/bin/httpd
command-line = $${:apache-executable} -f $${cgi-httpd-configuration-file:output} -DFOREGROUND
wrapper-path = $${ca-httpd:executable}
[cgi-httpd-graceful-wrapper]
recipe = slapos.cookbook:wrapper
command-line = kill -USR1 $(cat $${cgi-httpd-configuration-file:pid-file})
wrapper-path = $${monitor-directory:etc-run}/cgi-httpd-graceful
[monitor-promise]
recipe = slapos.cookbook:check_url_available
path = $${directory:promises}/monitor
url = https://[$${cgi-httpd-configuration-file:listening-ip}]:$${monitor-parameters:port}/$${deploy-index:filename}
path = $${monitor-directory:promise}/monitor
url = $${monitor-parameters:url}/$${monitor-parameters:index-filename}
check-secure = 1
dash_path = ${dash:location}/bin/dash
curl_path = ${curl:location}/bin/curl
[publish-connection-informations]
recipe = slapos.cookbook:publish
monitor_url = https://[$${cgi-httpd-configuration-file:listening-ip}]:$${monitor-parameters:port}
IMPORTANT_monitor_info = Change the monitor_password as soon as possible ! Default is : $${public:monitor-password}
monitor_url = $${monitor-parameters:url}
IMPORTANT_monitor_info = Change the monitor_password as soon as possible ! Default is : $${public:monitor-password} . You can change it in the setting.cgi section of your monitorin interface
stack/monitor/monitor.py.in
View file @ 82d2d645
......@@ -5,16 +5,21 @@ import json
import os
import subprocess
import sys
import sqlite3
import time
from optparse import OptionParser, make_option
FAILURE = "FAILURE"
SUCCESS = "SUCCESS"
db_path = "{{ monitor_parameter['db-path'] }}"
instance_path = "{{ directory['home'] }}"
monitor_dir = "{{ directory['monitor'] }}"
monitor_dir = "{{ directory['monitor-custom-scripts'] }}"
pid_dir = "{{ directory['run'] }}"
promise_dir = "{{ directory['promise'] }}"
monitoring_file_json = "{{ monitoring_file_json }}"
monitoring_folder_bool = "{{ monitoring_folder_bool }}"
option_list = [
make_option("-a", "--all", action="store_true", dest="all",
......@@ -30,16 +35,38 @@ option_list = [
]
def init_db():
db = sqlite3.connect(db_path)
c = db.cursor()
c.executescript("""
CREATE TABLE IF NOT EXISTS status (
timestamp INTEGER UNIQUE,
status VARCHAR(255));
CREATE TABLE IF NOT EXISTS individual_status (
timestamp INTEGER,
status VARCHAR(255),
element VARCHAR(255),
output TEXT);
""")
db.commit()
db.close()
def getListOfScripts(directory):
"""
Get the list of script inside of a directory (not recursive)
"""
scripts = []
if os.path.exists(directory) and os.path.isdir(directory):
for file in os.listdir(directory):
scripts.append(os.path.join(directory, file))
for file_name in os.listdir(directory):
file = os.path.join(directory, file_name)
if os.access(file, os.X_OK) and not os.path.isdir(file):
scripts.append(file)
else:
exit("There is a problem in your directories" \
"of monitoring. Please check them")
return scripts
def runServices(directory):
services = getListOfScripts(directory)
result = {}
......@@ -49,7 +76,8 @@ def runServices(directory):
try:
pid = int(open(service_path).read())
### because apache (or others) can write sockets
except IOError:
### We also ignore not readable pid files
except (IOError, ValueError):
continue
try:
os.kill(pid, 0)
......@@ -91,22 +119,28 @@ def runScripts(directory):
def writeFiles(monitors):
timestamp = int(time.time())
date = datetime.datetime.now().ctime()
init_db()
db = sqlite3.connect(db_path)
fail = False
for i in monitors.values():
if i != "" :
for key, value in monitors.iteritems():
element_status = SUCCESS
if value != "" :
fail = True
element_status = FAILURE
db.execute("insert into individual_status(timestamp, element, output, status) values (?, ?, ?, ?)", (timestamp, key, value, element_status))
db.commit()
status = SUCCESS
if fail:
message = "FAILURE : something went wrong\n"
else:
message = "SUCCESS : everything is ok\n"
date = datetime.datetime.now().ctime()
status = FAILURE
db.execute("insert into status(timestamp, status) values (?, ?)", (timestamp, status))
db.commit()
db.close()
monitors['datetime'] = date
file_bool = os.path.join(monitoring_folder_bool, str(time.time()))
open(file_bool, "w+").write(date + "," + message)
open(monitoring_file_json, "w+").write(json.dumps(monitors))
if __name__ == "__main__":
def main():
parser = OptionParser(option_list=option_list)
monitors = {}
(options, args) = parser.parse_args()
......@@ -131,3 +165,6 @@ if __name__ == "__main__":
else:
exit(1)
if __name__ == "__main__":
main()
stack/monitor/status2rss.py
View file @ 82d2d645
import datetime
import PyRSS2Gen
import sys
from email.utils import parsedate_tz, mktime_tz
import sqlite3
import time
import base64
# Based on http://thehelpfulhacker.net/2011/03/27/a-rss-feed-for-your-crontabs/
......@@ -9,27 +10,36 @@ import base64
# ### Defaults
TITLE = sys.argv[1]
LINK = sys.argv[2]
db_path = sys.argv[3]
DESCRIPTION = TITLE
SUCCESS = "SUCCESS"
FAILURE = "FAILURE"
items = []
status = ""
while 1:
try:
line = sys.stdin.readline()
except KeyboardInterrupt:
break
current_timestamp = int(time.time())
# We only build the RSS for the last ten days
period = 3600 * 24 * 10
db = sqlite3.connect(db_path)
rows = db.execute("select timestamp, status from status where timestamp>? order by timestamp", (current_timestamp - period,))
for row in rows:
line_timestamp, line_status = row
line_status = line_status.encode()
if not line:
break
if line_status == status:
continue
time, desc = line.split(',', 1)
status = line_status
event_time = datetime.datetime.fromtimestamp(line_timestamp).strftime('%Y-%m-%d %H:%M:%S')
rss_item = PyRSS2Gen.RSSItem(
title = desc,
description = "%s, %s" % (time, desc),
title = status,
description = "%s: %s" % (event_time, status),
link = LINK,
pubDate = datetime.datetime.fromtimestamp(mktime_tz(parsedate_tz(time))),
guid = PyRSS2Gen.Guid(base64.b64encode("%s, %s" % (time, desc)))
pubDate = event_time,
guid = PyRSS2Gen.Guid(base64.b64encode("%s, %s" % (event_time, status)))
)
items.append(rss_item)
......
stack/monitor/webfiles/index.html.jinja2
View file @ 82d2d645
......@@ -17,6 +17,8 @@
<li><a href="{{ category }}/{{ script }}" class="script">{{ script }}</a></li>
{% endfor %}
{% endfor %}
<li class="pure-menu-heading category">Files</li>
<li><a href="./private/" class="link"> User: admin</br> Password is yours</a></li>
</ul>
</div>
</div>
......
stack/monitor/webfiles/status.cgi.in
View file @ 82d2d645
#!{{ python_executable }}
import cgi
import cgitb
import json
import subprocess
def refresh():
command = ["{{ monitor_bin }}", "-a"]
subprocess.call(command)
cgitb.enable(display=0, logdir="/tmp/cgi.log")
form = cgi.FieldStorage()
if "refresh" in form:
refresh()
json_file = "{{ json_file }}"
result = json.load(open(json_file))
......@@ -13,8 +22,12 @@ print "<link rel=\"stylesheet\" href=\"pure-min.css\">"
print "<link rel=\"stylesheet\" href=\"/style.css\">"
print "</head><body>"
print "<h1>Monitoring :</h1>"
print "<form action=\"/index.cgi\" method=\"post\" class=\"pure-form-aligned\">"
print "<input type=\"hidden\" name=\"posting-script\" value=\"{{ pwd }}/{{ this_file }}\">"
print "<p><em>Last time of monitoring process : %s</em></p>" % (result['datetime'])
del result['datetime']
print "<div class=\"pure-controls\"><button type=\"submit\" class=\"pure-button \
pure-button-primary\" name=\"refresh\" value=\"refresh\">Refresh</button></div></form>"
print "<br/>"
print "<h2>These scripts and promises have failed :</h2>"
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment
GitLab Nexedi Edition | About GitLab | About Nexedi | 沪ICP备2021021310号-2 | 沪ICP备2021021310号-7