Skip to content
Projects
Groups
Snippets
Help
Loading...
Help
Support
Keyboard shortcuts
?
Submit feedback
Contribute to GitLab
Sign in / Register
Toggle navigation
slapos
Project overview
Project overview
Details
Activity
Releases
Repository
Repository
Files
Commits
Branches
Tags
Contributors
Graph
Compare
Issues
0
Issues
0
List
Boards
Labels
Milestones
Merge Requests
0
Merge Requests
0
CI / CD
CI / CD
Pipelines
Jobs
Schedules
Analytics
Analytics
CI / CD
Repository
Value Stream
Wiki
Wiki
Snippets
Snippets
Members
Members
Collapse sidebar
Close sidebar
Activity
Graph
Create a new issue
Jobs
Commits
Issue Boards
Open sidebar
Paul Graydon
slapos
Commits
ed459ed1
Commit
ed459ed1
authored
Apr 22, 2016
by
Alain Takoudjou
Browse files
Options
Browse Files
Download
Plain Diff
Merge branch 'master' into 1.0
parents
3c897682
7bdaefa1
Changes
15
Hide whitespace changes
Inline
Side-by-side
Showing
15 changed files
with
106 additions
and
110 deletions
+106
-110
component/dropbear/buildout.cfg
component/dropbear/buildout.cfg
+3
-1
slapos/recipe/dropbear.py
slapos/recipe/dropbear.py
+1
-4
slapos/recipe/librecipe/generic.py
slapos/recipe/librecipe/generic.py
+27
-18
slapos/recipe/wrapper.py
slapos/recipe/wrapper.py
+12
-4
software/apache-frontend/common.cfg
software/apache-frontend/common.cfg
+4
-4
software/apache-frontend/instance-apache-frontend.cfg
software/apache-frontend/instance-apache-frontend.cfg
+10
-43
software/apache-frontend/instance-apache-replicate.cfg.in
software/apache-frontend/instance-apache-replicate.cfg.in
+17
-9
software/apache-frontend/instance.cfg
software/apache-frontend/instance.cfg
+2
-0
software/apache-frontend/templates/apache-custom-slave-list.cfg.in
...apache-frontend/templates/apache-custom-slave-list.cfg.in
+1
-1
software/kvm/instance-kvm-resilient-input-schema.json
software/kvm/instance-kvm-resilient-input-schema.json
+2
-2
software/slaprunner/instance-runner-input-schema.json
software/slaprunner/instance-runner-input-schema.json
+11
-16
software/slaprunner/instance-runner-resilient-input-schema.json
...re/slaprunner/instance-runner-resilient-input-schema.json
+3
-2
stack/monitor/buildout.cfg
stack/monitor/buildout.cfg
+2
-2
stack/monitor/instance-monitor.cfg.jinja2.in
stack/monitor/instance-monitor.cfg.jinja2.in
+3
-1
stack/monitor/scripts/monitor.py
stack/monitor/scripts/monitor.py
+8
-3
No files found.
component/dropbear/buildout.cfg
View file @
ed459ed1
...
...
@@ -20,9 +20,11 @@ md5sum = 0284ea239083f04c8b874e08e1aca243
# in order have all patches working.
url = http://matt.ucc.asn.au/dropbear/releases/dropbear-0.53.1.tar.bz2
# NOTE DEFAULT_RECV_WINDOW and RECV_MAX_PAYLOAD_LEN are tweaked to support
# faster network throughput compared to dropbear defaults.
configure-options =
--with-zlib=${zlib:location}
CFLAGS="-DENABLE_SINGLEUSER -D__DIRTY_NO_SHELL_CHECKING"
CFLAGS="-DENABLE_SINGLEUSER -D__DIRTY_NO_SHELL_CHECKING
-DDEFAULT_RECV_WINDOW=1048576 -DRECV_MAX_PAYLOAD_LEN=524288
"
environment =
CPPFLAGS =-I${zlib:location}/include
...
...
slapos/recipe/dropbear.py
View file @
ed459ed1
...
...
@@ -182,15 +182,12 @@ def keysplit(s):
class
AddAuthorizedKey
(
GenericBaseRecipe
):
def
install
(
self
):
path_list
=
[]
ssh
=
self
.
createDirectory
(
self
.
options
[
'home'
],
'.ssh'
)
path_list
.
append
(
ssh
)
authorized_keys
=
AuthorizedKeysFile
(
os
.
path
.
join
(
ssh
,
'authorized_keys'
))
for
key
in
keysplit
(
self
.
options
[
'key'
]):
# XXX key might actually be the string 'None' or 'null'
authorized_keys
.
append
(
key
)
return
path_list
return
[
authorized_keys
.
filename
]
slapos/recipe/librecipe/generic.py
View file @
ed459ed1
...
...
@@ -134,23 +134,25 @@ class GenericBaseRecipe(object):
pidfile
=
None
):
"""
Creates a
very simple (one command)
shell script for process replacement.
Creates a shell script for process replacement.
Takes care of quoting.
Takes care of #! line limitation when the wrapped command is a script.
if pidfile parameter is specified, then it will make the wrapper a singleton,
accepting to run only if no other instance is running.
"""
lines
=
[
'#!/bin/sh'
]
for
comment
in
comments
:
lines
.
append
(
'# %s'
%
comment
)
if
comments
:
lines
+=
'# '
,
'
\
n
# '
.
join
(
comments
),
'
\
n
'
if
environment
:
for
key
in
environment
:
lines
.
append
(
'export %s=%s'
%
(
key
,
environment
[
key
]))
lines
.
append
(
'COMMAND='
+
shlex
.
quote
(
command
))
for
key
in
environment
or
():
lines
.
append
(
'export %s=%s'
%
(
key
,
environment
[
key
]))
if
pidfile
:
lines
.
append
(
dedent
(
"""
\
lines
.
append
(
dedent
(
"""
# Check for other instances
pidfile=%s
if [ -e $pidfile ]; then
...
...
@@ -162,24 +164,31 @@ class GenericBaseRecipe(object):
rm $pidfile
fi
fi
echo $$ > $pidfile"""
%
(
pidfile
,
command
)))
echo $$ > $pidfile"""
%
(
shlex
.
quote
(
pidfile
),
command
)))
lines
.
append
(
dedent
(
'''
# If the wrapped command uses a shebang, execute the referenced
# executable passing the script path as first argument.
# This is to workaround the limitation of 127 characters in #!
[ ! -f "$COMMAND" ] || {
[ "`head -c2`" != "#!" ] || read -r EXE ARG
} < "$COMMAND"
lines
.
append
(
'exec %s'
%
shlex
.
quote
(
command
))
exec $EXE ${ARG:+"$ARG"} "$COMMAND"'''
))
parameters
=
map
(
shlex
.
quote
,
parameters
)
if
parameters_extra
:
# pass-through further parameters
parameters
.
append
(
'"$@"'
)
for
param
in
parameters
:
if
len
(
lines
[
-
1
])
<
40
:
lines
[
-
1
]
+=
' '
+
shlex
.
quote
(
param
)
lines
[
-
1
]
+=
' '
+
param
else
:
lines
[
-
1
]
+=
'
\
\
'
lines
.
append
(
'
\
t
'
+
shlex
.
quote
(
param
))
if
parameters_extra
:
# pass-through further parameters
lines
[
-
1
]
+=
'
\
\
'
lines
.
append
(
'
\
t
"$@"'
)
lines
.
append
(
'
\
t
'
+
param
)
content
=
'
\
n
'
.
join
(
lines
)
+
'
\
n
'
return
self
.
createFile
(
name
,
content
,
0700
)
lines
.
append
(
''
)
return
self
.
createFile
(
name
,
'
\
n
'
.
join
(
lines
)
,
0700
)
def
createDirectory
(
self
,
parent
,
name
,
mode
=
0700
):
path
=
os
.
path
.
join
(
parent
,
name
)
...
...
slapos/recipe/wrapper.py
View file @
ed459ed1
...
...
@@ -54,8 +54,16 @@ class Recipe(GenericBaseRecipe):
if
environment
is
not
None
:
environment
=
dict
((
k
.
strip
(),
v
.
strip
())
for
k
,
v
in
[
line
.
split
(
'='
)
for
line
in
environment
.
splitlines
()
if
line
.
strip
()
])
return
[
self
.
createPythonScript
(
wrapper_path
,
# We create a python script and a wrapper around the python
# script because the python script might have a too long #! line
python_script
=
self
.
createPythonScript
(
wrapper_path
+
'.py'
,
'slapos.recipe.librecipe.execute.generic_exec'
,
(
command_line
,
wait_files
,
environment
,),
)]
(
command_line
,
wait_files
,
environment
,),
)
return
[
python_script
,
self
.
createWrapper
(
name
=
wrapper_path
,
command
=
python_script
,
parameters
=
[],
parameters_extra
=
parameters_extra
)
]
software/apache-frontend/common.cfg
View file @
ed459ed1
...
...
@@ -58,27 +58,27 @@ command =
[template]
recipe = slapos.recipe.template
url = ${:_profile_base_location_}/instance.cfg
md5sum =
16b22ef9b0476352e2e5b68f5966ce62
md5sum =
51752d0e4eae8c761750117c44983594
output = ${buildout:directory}/template.cfg
mode = 0644
[template-apache-frontend]
recipe = slapos.recipe.template
url = ${:_profile_base_location_}/instance-apache-frontend.cfg
md5sum = 0
0361d1e4a2788863de58e0ca9b6dfcf
md5sum = 0
7b6f864b46cb2f7b4b8aab71da95d13
output = ${buildout:directory}/template-apache-frontend.cfg
mode = 0644
[template-apache-replicate]
recipe = slapos.recipe.build:download
url = ${:_profile_base_location_}/instance-apache-replicate.cfg.in
md5sum =
c495395bf4cae93ff665c5e74e6d4583
md5sum =
a05bc0f925902ddf0b4d12146b10c9e1
mode = 0644
[template-slave-list]
recipe = slapos.recipe.build:download
url = ${:_profile_base_location_}/templates/apache-custom-slave-list.cfg.in
md5sum =
f09759a0b68337d820c70e72afb4dbf9
md5sum =
c01c0ebb749d16bda4649bf42d36c7ea
mode = 640
[template-slave-configuration]
...
...
software/apache-frontend/instance-apache-frontend.cfg
View file @
ed459ed1
...
...
@@ -33,28 +33,9 @@ parts =
trafficserver-storage-config
trafficserver-promise-listen-port
## Monitoring part
###Parts to add for monitoring
certificate-authority
cron-entry-monitor
cron-entry-rss
deploy-index
deploy-settings-cgi
deploy-status-cgi
deploy-status-history-cgi
setup-static-files
certificate-authority
zero-parameters
public-symlink
cgi-httpd-wrapper
cgi-httpd-graceful-wrapper
monitor-promise
monitor-instance-log-access
## Monitor for apache
monitor-current-log-access
monitor-backup-log-access
monitor-base
monitor-ats-cache-stats-wrapper
monitor-apache-configuration-verification
monitor-verify-re6st-connectivity
extends = ${monitor-template:output}
...
...
@@ -74,9 +55,9 @@ srv = $${buildout:directory}/srv/
var = $${buildout:directory}/var/
template = $${buildout:directory}/template/
backup = $${
:srv}
/backup
log = $${
:var}
/log
run = $${
:var}
/run
backup = $${
buildout:directory}/srv
/backup
log = $${
buildout:directory}/var
/log
run = $${
buildout:directory}/var
/run
service = $${:etc}/service
etc-run = $${:etc}/run
promise = $${:etc}/promise
...
...
@@ -181,7 +162,7 @@ extra-context =
section logrotate_dict logrotate
section frontend_configuration frontend-configuration
section apache_configuration apache-configuration
key monitor_
url monitor-parameters:url
key monitor_
base_url monitor-instance-parameter:monitor-base-url
[dynamic-custom-group-template-slave-list]
< = jinja2-template-base
...
...
@@ -633,19 +614,15 @@ frontend-name =
# Monitoring sections
#
[monitor-current-log-access]
< = monitor-directory-access
source = $${directory:log}
[monitor-backup-log-access]
< = monitor-directory-access
source = $${directory:logrotate-backup}
[monitor-conf-parameters]
private-path-list +=
$${directory:logrotate-backup}
# Produce ATS Cache stats
[monitor-ats-cache-stats-wrapper]
< = jinja2-template-base
template = ${template-wrapper:output}
rendered = $${monitor-directory:
monitoring-cgi
}/ats-cache-stats
rendered = $${monitor-directory:
report
}/ats-cache-stats
mode = 0700
command = export TS_ROOT=$${buildout:directory} && echo "<pre>$(${trafficserver:location}/bin/traffic_shell $${monitor-ats-cache-stats-config:rendered})</pre>"
extra-context =
...
...
@@ -659,19 +636,9 @@ mode = 644
context =
raw content show:cache-stats
# Display result of apache configuration check
[monitor-apache-configuration-verification]
< = jinja2-template-base
template = ${template-wrapper:output}
rendered = $${monitor-directory:monitoring-cgi}/front-httpd-configuration
mode = 0700
command = echo "<pre>$($${apache-configuration:frontend-configuration-verification})</pre>"
extra-context =
key content :command
[monitor-verify-re6st-connectivity]
recipe = slapos.cookbook:check_url_available
path = $${
monitor-directory:monitor-custom-scripts
}/re6st-connectivity
path = $${
directory:promise
}/re6st-connectivity
url = $${public:re6st-verification-url}
dash_path = ${dash:location}/bin/dash
curl_path = ${curl:location}/bin/curl
...
...
software/apache-frontend/instance-apache-replicate.cfg.in
View file @
ed459ed1
...
...
@@ -80,6 +80,10 @@ context =
[replicate]
<= slap-connection
recipe = slapos.cookbook:requestoptional
config-monitor-cors-domains = {{ slapparameter_dict.get('monitor-cors-domains', 'monitor.app.officejs.com') }}
config-monitor-username = ${monitor-htpasswd:username}
config-monitor-password = ${monitor-htpasswd:passwd}
{% set frontend_software_url_key = "-frontend-software-release-url" %}
{% if slapparameter_dict.has_key(frontend_software_url_key) %}
software-url = {{ slapparameter_dict.pop(frontend_software_url_key) }}
...
...
@@ -87,8 +91,7 @@ software-url = {{ slapparameter_dict.pop(frontend_software_url_key) }}
software-url = ${slap-connection:software-release-url}
{% endif %}
software-type = {{frontend_type}}
return = private-ipv4 public-ipv4 slave-instance-information-list monitor_url
connection-monitor_url =
return = private-ipv4 public-ipv4 slave-instance-information-list monitor-base-url
{% for section, frontend_request in request_dict.iteritems() %}
[{{section}}]
...
...
@@ -117,13 +120,10 @@ slave-amount = {{ slave_instance_list | length }}
accepted-slave-amount = {{ authorized_slave_list | length }}
rejected-slave-amount = {{ rejected_slave_list | length }}
rejected-slave-list = {{ json_module.dumps(rejected_slave_list) }}
{% for frontend in frontend_section_list %}
{{ frontend }}-monitor-url = {{ '${' + frontend + ':connection-monitor_url}' }}
{% endfor -%}
{% for frontend in frontend_list -%}
#{{frontend}}-private-ipv4 = ${request-{{frontend}}:private-ipv4}
{% endfor -%}
monitor-base-url = ${monitor-conf-parameters:base-url}
monitor-url = ${:monitor-base-url}/public/feeds
monitor-user = ${monitor-instance-parameter:username}
monitor-password = ${monitor-instance-parameter:password}
#----------------------------
#--
...
...
@@ -149,8 +149,16 @@ extensions = jinja2.ext.do
extra-context =
section slave_information slave-information
[monitor-conf-parameters]
monitor-url-list +=
{% for frontend in frontend_section_list %}
{{ ' ${' + frontend + ':connection-monitor-base-url}' }}
{% endfor -%}
[buildout]
extends = {{ template_monitor }}
parts =
monitor-base
publish-slave-information
publish-information
{% for part in part_list -%}
...
...
software/apache-frontend/instance.cfg
View file @
ed459ed1
...
...
@@ -48,3 +48,5 @@ extra-context =
raw template_publish_slave_information ${template-replicate-publish-slave-information:target}
# Must match the key id in [switch-softwaretype] which uses this section.
raw software_type RootSoftwareInstance-default-custom-personal-custom-group-replicate
raw template_monitor ${monitor2-template:rendered}
software/apache-frontend/templates/apache-custom-slave-list.cfg.in
View file @
ed459ed1
...
...
@@ -262,7 +262,7 @@ private-ipv4 = {{ local_ipv4 }}
{% if extra_slave_instance_list -%}
slave-instance-information-list = {{ json_module.dumps(slave_instance_information_list) }}
{% endif -%}
monitor
_url = {{ monitor
_url }}
monitor
-base-url = {{ monitor_base
_url }}
[buildout]
parts +=
...
...
software/kvm/instance-kvm-resilient-input-schema.json
View file @
ed459ed1
...
...
@@ -35,14 +35,14 @@
"title"
:
"Remove backups older than..."
,
"description"
:
"Remove all the backups in PBS that are older than specified value. It should be rdiff-backup-compatible."
,
"type"
:
"string"
,
"default"
:
"
3B
"
,
"default"
:
"
2W
"
,
"optional"
:
true
},
"resilient-clone-number"
:
{
"title"
:
"Amount of backup(s) to create"
,
"description"
:
"Amount of backup(s) to create. Each backup consists of a Pull Backup Server and a clone."
,
"type"
:
"integer"
,
"default"
:
2
,
"default"
:
1
,
"optional"
:
true
},
"ignore-known-hosts-file"
:
{
...
...
software/slaprunner/instance-runner-input-schema.json
View file @
ed459ed1
...
...
@@ -50,6 +50,17 @@
"description"
:
"Software type of your instance inside the runner"
,
"type"
:
"string"
},
"cpu-usage-ratio"
:
{
"title"
:
"CPU Usage Ratio"
,
"description"
:
"Ratio of the CPU use for compilation, if value is set to n, compilation will use number-of-cpu/n of cpus (need instance restart)"
,
"type"
:
"integer"
,
"default"
:
4
},
"instance-name"
:
{
"title"
:
"Displayed instance name"
,
"description"
:
"Name of the instance, to show in the window title"
,
"type"
:
"string"
},
"custom-frontend-backend-url"
:
{
"title"
:
"Custom Frontend Backend URL"
,
"description"
:
"return an ipv4 frontend of the given ipv6(+optional port)"
,
...
...
@@ -110,22 +121,6 @@
"description"
:
"List of cors domains separated with space. Needed for ajax query on this monitor instance from a different domain."
,
"type"
:
"string"
,
"default"
:
"monitor.app.officejs.com"
},
"cpu-usage-ratio"
:
{
"title"
:
"CPU Usage Ratio"
,
"description"
:
"Ratio of the CPU use for compilation, if value is set to n, compilation will use number-of-cpu/n of cpus (need instance restart)"
,
"type"
:
"integer"
,
"default"
:
4
},
"resilient-clone-number"
:
{
"title"
:
"Number of Resilient Clones"
,
"description"
:
"Number of clones to be deployed when a resilient runner, if missing a single clone is provided. Its value can be 1 or 2"
,
"type"
:
"integer"
,
"enum"
:
[
1
,
2
]
},
"instance-name"
:
{
"description"
:
"Name of the instance, to show in the window title"
,
"type"
:
"string"
}
}
}
software/slaprunner/instance-runner-resilient-input-schema.json
View file @
ed459ed1
...
...
@@ -35,14 +35,15 @@
"title"
:
"Remove backups older than..."
,
"description"
:
"Remove all the backups in PBS that are older than specified value. It should be rdiff-backup-compatible."
,
"type"
:
"string"
,
"default"
:
"
3B
"
,
"default"
:
"
2W
"
,
"optional"
:
true
},
"resilient-clone-number"
:
{
"title"
:
"Amount of backup(s) to create"
,
"description"
:
"Amount of backup(s) to create. Each backup consists of a Pull Backup Server and a clone."
,
"type"
:
"integer"
,
"default"
:
2
,
"default"
:
1
,
"enum"
:
[
1
,
2
],
"optional"
:
true
},
"ignore-known-hosts-file"
:
{
...
...
stack/monitor/buildout.cfg
View file @
ed459ed1
...
...
@@ -99,7 +99,7 @@ recipe = slapos.recipe.template:jinja2
filename = template-monitor.cfg
template = ${:_profile_base_location_}/instance-monitor.cfg.jinja2.in
rendered = ${buildout:directory}/template-monitor.cfg
md5sum =
eab45fd2b0f017ab97a9c8f466c8dceb
md5sum =
cd7f386fe1b5066d8508758249b408d3
context =
key apache_location apache:location
key gzip_location gzip:location
...
...
@@ -130,7 +130,7 @@ depends =
[monitor2-bin]
<= monitor-template-script
filename = monitor.py
md5sum =
222365a469f8ab08a0367d81c0b03982
md5sum =
31beec15d3c3cd7979d04ecb834c439a
[run-promise-py]
recipe = slapos.recipe.template:jinja2
...
...
stack/monitor/instance-monitor.cfg.jinja2.in
View file @
ed459ed1
...
...
@@ -61,6 +61,7 @@ services = ${:etc}/service
promises = ${:etc}/promise
monitor = ${:srv}/monitor
monitor-promise = ${:etc}/monitor-promise
monitor-report = ${:etc}/monitor-report
[monitor-directory]
recipe = slapos.cookbook:mkdirectory
...
...
@@ -69,6 +70,7 @@ etc = ${directory:etc}
run = ${directory:monitor}/run
#run = ${directory:scripts}
promises = ${directory:monitor-promise}
report = ${directory:monitor-report}
pids = ${directory:run}/monitor
cgi-bin = ${directory:monitor}/cgi-bin
webdav = ${directory:monitor}/webdav
...
...
@@ -374,7 +376,7 @@ monitor-httpd-port = 8196
monitor-base-url = ${monitor-frontend-promise:url}
root-instance-title = ${slap-configuration:root-instance-title}
opml-url-list =
cors-domains =
cors-domains =
monitor.app.officejs.com
# XXX Hard coded parameter
collector-db = /srv/slapgrid/var/data-log/collector.db
# Credentials
...
...
stack/monitor/scripts/monitor.py
View file @
ed459ed1
...
...
@@ -222,13 +222,18 @@ class Monitoring(object):
def
getMonitorTitleFromUrl
(
self
,
monitor_url
):
# This file should be generated
if
not
monitor_url
.
startswith
(
'https://'
)
and
not
monitor_url
.
startswith
(
'http://'
):
return
'Unknow Instance'
return
'Unknow
n
Instance'
if
not
monitor_url
.
endswith
(
'/'
):
monitor_url
=
monitor_url
+
'/'
context
=
ssl
.
_create_unverified_context
()
url
=
monitor_url
+
'/.jio_documents/monitor.global.json'
# XXX Hard Coded path
try
:
response
=
urllib2
.
urlopen
(
url
,
context
=
context
)
# XXX - working here with public url
if
hasattr
(
ssl
,
'_create_unverified_context'
):
context
=
ssl
.
_create_unverified_context
()
response
=
urllib2
.
urlopen
(
url
,
context
=
context
)
else
:
response
=
urllib2
.
urlopen
(
url
)
except
urllib2
.
HTTPError
:
return
'Unknow Instance'
else
:
...
...
Write
Preview
Markdown
is supported
0%
Try again
or
attach a new file
Attach a file
Cancel
You are about to add
0
people
to the discussion. Proceed with caution.
Finish editing this message first!
Cancel
Please
register
or
sign in
to comment