Skip to content
Projects
Groups
Snippets
Help
Loading...
Help
Support
Keyboard shortcuts
?
Submit feedback
Contribute to GitLab
Sign in / Register
Toggle navigation
slapos
Project overview
Project overview
Details
Activity
Releases
Repository
Repository
Files
Commits
Branches
Tags
Contributors
Graph
Compare
Issues
0
Issues
0
List
Boards
Labels
Milestones
Merge Requests
0
Merge Requests
0
CI / CD
CI / CD
Pipelines
Jobs
Schedules
Analytics
Analytics
CI / CD
Repository
Value Stream
Wiki
Wiki
Snippets
Snippets
Members
Members
Collapse sidebar
Close sidebar
Activity
Graph
Create a new issue
Jobs
Commits
Issue Boards
Open sidebar
Lu Xu
slapos
Commits
ed459ed1
Commit
ed459ed1
authored
8 years ago
by
Alain Takoudjou
Browse files
Options
Browse Files
Download
Plain Diff
Merge branch 'master' into 1.0
parents
3c897682
7bdaefa1
Changes
15
Show whitespace changes
Inline
Side-by-side
Showing
15 changed files
with
106 additions
and
110 deletions
+106
-110
component/dropbear/buildout.cfg
component/dropbear/buildout.cfg
+3
-1
slapos/recipe/dropbear.py
slapos/recipe/dropbear.py
+1
-4
slapos/recipe/librecipe/generic.py
slapos/recipe/librecipe/generic.py
+27
-18
slapos/recipe/wrapper.py
slapos/recipe/wrapper.py
+12
-4
software/apache-frontend/common.cfg
software/apache-frontend/common.cfg
+4
-4
software/apache-frontend/instance-apache-frontend.cfg
software/apache-frontend/instance-apache-frontend.cfg
+10
-43
software/apache-frontend/instance-apache-replicate.cfg.in
software/apache-frontend/instance-apache-replicate.cfg.in
+17
-9
software/apache-frontend/instance.cfg
software/apache-frontend/instance.cfg
+2
-0
software/apache-frontend/templates/apache-custom-slave-list.cfg.in
...apache-frontend/templates/apache-custom-slave-list.cfg.in
+1
-1
software/kvm/instance-kvm-resilient-input-schema.json
software/kvm/instance-kvm-resilient-input-schema.json
+2
-2
software/slaprunner/instance-runner-input-schema.json
software/slaprunner/instance-runner-input-schema.json
+11
-16
software/slaprunner/instance-runner-resilient-input-schema.json
...re/slaprunner/instance-runner-resilient-input-schema.json
+3
-2
stack/monitor/buildout.cfg
stack/monitor/buildout.cfg
+2
-2
stack/monitor/instance-monitor.cfg.jinja2.in
stack/monitor/instance-monitor.cfg.jinja2.in
+3
-1
stack/monitor/scripts/monitor.py
stack/monitor/scripts/monitor.py
+8
-3
No files found.
component/dropbear/buildout.cfg
View file @
ed459ed1
...
...
@@ -20,9 +20,11 @@ md5sum = 0284ea239083f04c8b874e08e1aca243
# in order have all patches working.
url = http://matt.ucc.asn.au/dropbear/releases/dropbear-0.53.1.tar.bz2
# NOTE DEFAULT_RECV_WINDOW and RECV_MAX_PAYLOAD_LEN are tweaked to support
# faster network throughput compared to dropbear defaults.
configure-options =
--with-zlib=${zlib:location}
CFLAGS="-DENABLE_SINGLEUSER -D__DIRTY_NO_SHELL_CHECKING"
CFLAGS="-DENABLE_SINGLEUSER -D__DIRTY_NO_SHELL_CHECKING
-DDEFAULT_RECV_WINDOW=1048576 -DRECV_MAX_PAYLOAD_LEN=524288
"
environment =
CPPFLAGS =-I${zlib:location}/include
...
...
This diff is collapsed.
Click to expand it.
slapos/recipe/dropbear.py
View file @
ed459ed1
...
...
@@ -182,15 +182,12 @@ def keysplit(s):
class
AddAuthorizedKey
(
GenericBaseRecipe
):
def
install
(
self
):
path_list
=
[]
ssh
=
self
.
createDirectory
(
self
.
options
[
'home'
],
'.ssh'
)
path_list
.
append
(
ssh
)
authorized_keys
=
AuthorizedKeysFile
(
os
.
path
.
join
(
ssh
,
'authorized_keys'
))
for
key
in
keysplit
(
self
.
options
[
'key'
]):
# XXX key might actually be the string 'None' or 'null'
authorized_keys
.
append
(
key
)
return
path_list
return
[
authorized_keys
.
filename
]
This diff is collapsed.
Click to expand it.
slapos/recipe/librecipe/generic.py
View file @
ed459ed1
...
...
@@ -134,23 +134,25 @@ class GenericBaseRecipe(object):
pidfile
=
None
):
"""
Creates a
very simple (one command)
shell script for process replacement.
Creates a shell script for process replacement.
Takes care of quoting.
Takes care of #! line limitation when the wrapped command is a script.
if pidfile parameter is specified, then it will make the wrapper a singleton,
accepting to run only if no other instance is running.
"""
lines
=
[
'#!/bin/sh'
]
for
comment
in
comments
:
lines
.
append
(
'# %s'
%
comment
)
if
comments
:
lines
+=
'# '
,
'
\
n
# '
.
join
(
comments
),
'
\
n
'
if
environment
:
for
key
in
environment
:
lines
.
append
(
'COMMAND='
+
shlex
.
quote
(
command
))
for
key
in
environment
or
():
lines
.
append
(
'export %s=%s'
%
(
key
,
environment
[
key
]))
if
pidfile
:
lines
.
append
(
dedent
(
"""
\
lines
.
append
(
dedent
(
"""
# Check for other instances
pidfile=%s
if [ -e $pidfile ]; then
...
...
@@ -162,24 +164,31 @@ class GenericBaseRecipe(object):
rm $pidfile
fi
fi
echo $$ > $pidfile"""
%
(
pidfile
,
command
)))
echo $$ > $pidfile"""
%
(
shlex
.
quote
(
pidfile
)
,
command
)))
lines
.
append
(
'exec %s'
%
shlex
.
quote
(
command
))
lines
.
append
(
dedent
(
'''
# If the wrapped command uses a shebang, execute the referenced
# executable passing the script path as first argument.
# This is to workaround the limitation of 127 characters in #!
[ ! -f "$COMMAND" ] || {
[ "`head -c2`" != "#!" ] || read -r EXE ARG
} < "$COMMAND"
for
param
in
parameters
:
if
len
(
lines
[
-
1
])
<
40
:
lines
[
-
1
]
+=
' '
+
shlex
.
quote
(
param
)
else
:
lines
[
-
1
]
+=
'
\
\
'
lines
.
append
(
'
\
t
'
+
shlex
.
quote
(
param
))
exec $EXE ${ARG:+"$ARG"} "$COMMAND"'''
))
parameters
=
map
(
shlex
.
quote
,
parameters
)
if
parameters_extra
:
# pass-through further parameters
parameters
.
append
(
'"$@"'
)
for
param
in
parameters
:
if
len
(
lines
[
-
1
])
<
40
:
lines
[
-
1
]
+=
' '
+
param
else
:
lines
[
-
1
]
+=
'
\
\
'
lines
.
append
(
'
\
t
"$@"'
)
lines
.
append
(
'
\
t
'
+
param
)
content
=
'
\
n
'
.
join
(
lines
)
+
'
\
n
'
return
self
.
createFile
(
name
,
content
,
0700
)
lines
.
append
(
''
)
return
self
.
createFile
(
name
,
'
\
n
'
.
join
(
lines
)
,
0700
)
def
createDirectory
(
self
,
parent
,
name
,
mode
=
0700
):
path
=
os
.
path
.
join
(
parent
,
name
)
...
...
This diff is collapsed.
Click to expand it.
slapos/recipe/wrapper.py
View file @
ed459ed1
...
...
@@ -54,8 +54,16 @@ class Recipe(GenericBaseRecipe):
if
environment
is
not
None
:
environment
=
dict
((
k
.
strip
(),
v
.
strip
())
for
k
,
v
in
[
line
.
split
(
'='
)
for
line
in
environment
.
splitlines
()
if
line
.
strip
()
])
return
[
self
.
createPythonScript
(
wrapper_path
,
# We create a python script and a wrapper around the python
# script because the python script might have a too long #! line
python_script
=
self
.
createPythonScript
(
wrapper_path
+
'.py'
,
'slapos.recipe.librecipe.execute.generic_exec'
,
(
command_line
,
wait_files
,
environment
,),
)]
(
command_line
,
wait_files
,
environment
,),
)
return
[
python_script
,
self
.
createWrapper
(
name
=
wrapper_path
,
command
=
python_script
,
parameters
=
[],
parameters_extra
=
parameters_extra
)
]
This diff is collapsed.
Click to expand it.
software/apache-frontend/common.cfg
View file @
ed459ed1
...
...
@@ -58,27 +58,27 @@ command =
[template]
recipe = slapos.recipe.template
url = ${:_profile_base_location_}/instance.cfg
md5sum =
16b22ef9b0476352e2e5b68f5966ce62
md5sum =
51752d0e4eae8c761750117c44983594
output = ${buildout:directory}/template.cfg
mode = 0644
[template-apache-frontend]
recipe = slapos.recipe.template
url = ${:_profile_base_location_}/instance-apache-frontend.cfg
md5sum = 0
0361d1e4a2788863de58e0ca9b6dfcf
md5sum = 0
7b6f864b46cb2f7b4b8aab71da95d13
output = ${buildout:directory}/template-apache-frontend.cfg
mode = 0644
[template-apache-replicate]
recipe = slapos.recipe.build:download
url = ${:_profile_base_location_}/instance-apache-replicate.cfg.in
md5sum =
c495395bf4cae93ff665c5e74e6d4583
md5sum =
a05bc0f925902ddf0b4d12146b10c9e1
mode = 0644
[template-slave-list]
recipe = slapos.recipe.build:download
url = ${:_profile_base_location_}/templates/apache-custom-slave-list.cfg.in
md5sum =
f09759a0b68337d820c70e72afb4dbf9
md5sum =
c01c0ebb749d16bda4649bf42d36c7ea
mode = 640
[template-slave-configuration]
...
...
This diff is collapsed.
Click to expand it.
software/apache-frontend/instance-apache-frontend.cfg
View file @
ed459ed1
...
...
@@ -33,28 +33,9 @@ parts =
trafficserver-storage-config
trafficserver-promise-listen-port
## Monitoring part
###Parts to add for monitoring
certificate-authority
cron-entry-monitor
cron-entry-rss
deploy-index
deploy-settings-cgi
deploy-status-cgi
deploy-status-history-cgi
setup-static-files
certificate-authority
zero-parameters
public-symlink
cgi-httpd-wrapper
cgi-httpd-graceful-wrapper
monitor-promise
monitor-instance-log-access
## Monitor for apache
monitor-current-log-access
monitor-backup-log-access
monitor-base
monitor-ats-cache-stats-wrapper
monitor-apache-configuration-verification
monitor-verify-re6st-connectivity
extends = ${monitor-template:output}
...
...
@@ -74,9 +55,9 @@ srv = $${buildout:directory}/srv/
var = $${buildout:directory}/var/
template = $${buildout:directory}/template/
backup = $${
:srv}
/backup
log = $${
:var}
/log
run = $${
:var}
/run
backup = $${
buildout:directory}/srv
/backup
log = $${
buildout:directory}/var
/log
run = $${
buildout:directory}/var
/run
service = $${:etc}/service
etc-run = $${:etc}/run
promise = $${:etc}/promise
...
...
@@ -181,7 +162,7 @@ extra-context =
section logrotate_dict logrotate
section frontend_configuration frontend-configuration
section apache_configuration apache-configuration
key monitor_
url monitor-parameters:url
key monitor_
base_url monitor-instance-parameter:monitor-base-url
[dynamic-custom-group-template-slave-list]
< = jinja2-template-base
...
...
@@ -633,19 +614,15 @@ frontend-name =
# Monitoring sections
#
[monitor-current-log-access]
< = monitor-directory-access
source = $${directory:log}
[monitor-backup-log-access]
< = monitor-directory-access
source = $${directory:logrotate-backup}
[monitor-conf-parameters]
private-path-list +=
$${directory:logrotate-backup}
# Produce ATS Cache stats
[monitor-ats-cache-stats-wrapper]
< = jinja2-template-base
template = ${template-wrapper:output}
rendered = $${monitor-directory:
monitoring-cgi
}/ats-cache-stats
rendered = $${monitor-directory:
report
}/ats-cache-stats
mode = 0700
command = export TS_ROOT=$${buildout:directory} && echo "<pre>$(${trafficserver:location}/bin/traffic_shell $${monitor-ats-cache-stats-config:rendered})</pre>"
extra-context =
...
...
@@ -659,19 +636,9 @@ mode = 644
context =
raw content show:cache-stats
# Display result of apache configuration check
[monitor-apache-configuration-verification]
< = jinja2-template-base
template = ${template-wrapper:output}
rendered = $${monitor-directory:monitoring-cgi}/front-httpd-configuration
mode = 0700
command = echo "<pre>$($${apache-configuration:frontend-configuration-verification})</pre>"
extra-context =
key content :command
[monitor-verify-re6st-connectivity]
recipe = slapos.cookbook:check_url_available
path = $${
monitor-directory:monitor-custom-scripts
}/re6st-connectivity
path = $${
directory:promise
}/re6st-connectivity
url = $${public:re6st-verification-url}
dash_path = ${dash:location}/bin/dash
curl_path = ${curl:location}/bin/curl
...
...
This diff is collapsed.
Click to expand it.
software/apache-frontend/instance-apache-replicate.cfg.in
View file @
ed459ed1
...
...
@@ -80,6 +80,10 @@ context =
[replicate]
<= slap-connection
recipe = slapos.cookbook:requestoptional
config-monitor-cors-domains = {{ slapparameter_dict.get('monitor-cors-domains', 'monitor.app.officejs.com') }}
config-monitor-username = ${monitor-htpasswd:username}
config-monitor-password = ${monitor-htpasswd:passwd}
{% set frontend_software_url_key = "-frontend-software-release-url" %}
{% if slapparameter_dict.has_key(frontend_software_url_key) %}
software-url = {{ slapparameter_dict.pop(frontend_software_url_key) }}
...
...
@@ -87,8 +91,7 @@ software-url = {{ slapparameter_dict.pop(frontend_software_url_key) }}
software-url = ${slap-connection:software-release-url}
{% endif %}
software-type = {{frontend_type}}
return = private-ipv4 public-ipv4 slave-instance-information-list monitor_url
connection-monitor_url =
return = private-ipv4 public-ipv4 slave-instance-information-list monitor-base-url
{% for section, frontend_request in request_dict.iteritems() %}
[{{section}}]
...
...
@@ -117,13 +120,10 @@ slave-amount = {{ slave_instance_list | length }}
accepted-slave-amount = {{ authorized_slave_list | length }}
rejected-slave-amount = {{ rejected_slave_list | length }}
rejected-slave-list = {{ json_module.dumps(rejected_slave_list) }}
{% for frontend in frontend_section_list %}
{{ frontend }}-monitor-url = {{ '${' + frontend + ':connection-monitor_url}' }}
{% endfor -%}
{% for frontend in frontend_list -%}
#{{frontend}}-private-ipv4 = ${request-{{frontend}}:private-ipv4}
{% endfor -%}
monitor-base-url = ${monitor-conf-parameters:base-url}
monitor-url = ${:monitor-base-url}/public/feeds
monitor-user = ${monitor-instance-parameter:username}
monitor-password = ${monitor-instance-parameter:password}
#----------------------------
#--
...
...
@@ -149,8 +149,16 @@ extensions = jinja2.ext.do
extra-context =
section slave_information slave-information
[monitor-conf-parameters]
monitor-url-list +=
{% for frontend in frontend_section_list %}
{{ ' ${' + frontend + ':connection-monitor-base-url}' }}
{% endfor -%}
[buildout]
extends = {{ template_monitor }}
parts =
monitor-base
publish-slave-information
publish-information
{% for part in part_list -%}
...
...
This diff is collapsed.
Click to expand it.
software/apache-frontend/instance.cfg
View file @
ed459ed1
...
...
@@ -48,3 +48,5 @@ extra-context =
raw template_publish_slave_information ${template-replicate-publish-slave-information:target}
# Must match the key id in [switch-softwaretype] which uses this section.
raw software_type RootSoftwareInstance-default-custom-personal-custom-group-replicate
raw template_monitor ${monitor2-template:rendered}
This diff is collapsed.
Click to expand it.
software/apache-frontend/templates/apache-custom-slave-list.cfg.in
View file @
ed459ed1
...
...
@@ -262,7 +262,7 @@ private-ipv4 = {{ local_ipv4 }}
{% if extra_slave_instance_list -%}
slave-instance-information-list = {{ json_module.dumps(slave_instance_information_list) }}
{% endif -%}
monitor
_url = {{ monitor
_url }}
monitor
-base-url = {{ monitor_base
_url }}
[buildout]
parts +=
...
...
This diff is collapsed.
Click to expand it.
software/kvm/instance-kvm-resilient-input-schema.json
View file @
ed459ed1
...
...
@@ -35,14 +35,14 @@
"title"
:
"Remove backups older than..."
,
"description"
:
"Remove all the backups in PBS that are older than specified value. It should be rdiff-backup-compatible."
,
"type"
:
"string"
,
"default"
:
"
3B
"
,
"default"
:
"
2W
"
,
"optional"
:
true
},
"resilient-clone-number"
:
{
"title"
:
"Amount of backup(s) to create"
,
"description"
:
"Amount of backup(s) to create. Each backup consists of a Pull Backup Server and a clone."
,
"type"
:
"integer"
,
"default"
:
2
,
"default"
:
1
,
"optional"
:
true
},
"ignore-known-hosts-file"
:
{
...
...
This diff is collapsed.
Click to expand it.
software/slaprunner/instance-runner-input-schema.json
View file @
ed459ed1
...
...
@@ -50,6 +50,17 @@
"description"
:
"Software type of your instance inside the runner"
,
"type"
:
"string"
},
"cpu-usage-ratio"
:
{
"title"
:
"CPU Usage Ratio"
,
"description"
:
"Ratio of the CPU use for compilation, if value is set to n, compilation will use number-of-cpu/n of cpus (need instance restart)"
,
"type"
:
"integer"
,
"default"
:
4
},
"instance-name"
:
{
"title"
:
"Displayed instance name"
,
"description"
:
"Name of the instance, to show in the window title"
,
"type"
:
"string"
},
"custom-frontend-backend-url"
:
{
"title"
:
"Custom Frontend Backend URL"
,
"description"
:
"return an ipv4 frontend of the given ipv6(+optional port)"
,
...
...
@@ -110,22 +121,6 @@
"description"
:
"List of cors domains separated with space. Needed for ajax query on this monitor instance from a different domain."
,
"type"
:
"string"
,
"default"
:
"monitor.app.officejs.com"
},
"cpu-usage-ratio"
:
{
"title"
:
"CPU Usage Ratio"
,
"description"
:
"Ratio of the CPU use for compilation, if value is set to n, compilation will use number-of-cpu/n of cpus (need instance restart)"
,
"type"
:
"integer"
,
"default"
:
4
},
"resilient-clone-number"
:
{
"title"
:
"Number of Resilient Clones"
,
"description"
:
"Number of clones to be deployed when a resilient runner, if missing a single clone is provided. Its value can be 1 or 2"
,
"type"
:
"integer"
,
"enum"
:
[
1
,
2
]
},
"instance-name"
:
{
"description"
:
"Name of the instance, to show in the window title"
,
"type"
:
"string"
}
}
}
This diff is collapsed.
Click to expand it.
software/slaprunner/instance-runner-resilient-input-schema.json
View file @
ed459ed1
...
...
@@ -35,14 +35,15 @@
"title"
:
"Remove backups older than..."
,
"description"
:
"Remove all the backups in PBS that are older than specified value. It should be rdiff-backup-compatible."
,
"type"
:
"string"
,
"default"
:
"
3B
"
,
"default"
:
"
2W
"
,
"optional"
:
true
},
"resilient-clone-number"
:
{
"title"
:
"Amount of backup(s) to create"
,
"description"
:
"Amount of backup(s) to create. Each backup consists of a Pull Backup Server and a clone."
,
"type"
:
"integer"
,
"default"
:
2
,
"default"
:
1
,
"enum"
:
[
1
,
2
],
"optional"
:
true
},
"ignore-known-hosts-file"
:
{
...
...
This diff is collapsed.
Click to expand it.
stack/monitor/buildout.cfg
View file @
ed459ed1
...
...
@@ -99,7 +99,7 @@ recipe = slapos.recipe.template:jinja2
filename = template-monitor.cfg
template = ${:_profile_base_location_}/instance-monitor.cfg.jinja2.in
rendered = ${buildout:directory}/template-monitor.cfg
md5sum =
eab45fd2b0f017ab97a9c8f466c8dceb
md5sum =
cd7f386fe1b5066d8508758249b408d3
context =
key apache_location apache:location
key gzip_location gzip:location
...
...
@@ -130,7 +130,7 @@ depends =
[monitor2-bin]
<= monitor-template-script
filename = monitor.py
md5sum =
222365a469f8ab08a0367d81c0b03982
md5sum =
31beec15d3c3cd7979d04ecb834c439a
[run-promise-py]
recipe = slapos.recipe.template:jinja2
...
...
This diff is collapsed.
Click to expand it.
stack/monitor/instance-monitor.cfg.jinja2.in
View file @
ed459ed1
...
...
@@ -61,6 +61,7 @@ services = ${:etc}/service
promises = ${:etc}/promise
monitor = ${:srv}/monitor
monitor-promise = ${:etc}/monitor-promise
monitor-report = ${:etc}/monitor-report
[monitor-directory]
recipe = slapos.cookbook:mkdirectory
...
...
@@ -69,6 +70,7 @@ etc = ${directory:etc}
run = ${directory:monitor}/run
#run = ${directory:scripts}
promises = ${directory:monitor-promise}
report = ${directory:monitor-report}
pids = ${directory:run}/monitor
cgi-bin = ${directory:monitor}/cgi-bin
webdav = ${directory:monitor}/webdav
...
...
@@ -374,7 +376,7 @@ monitor-httpd-port = 8196
monitor-base-url = ${monitor-frontend-promise:url}
root-instance-title = ${slap-configuration:root-instance-title}
opml-url-list =
cors-domains =
cors-domains =
monitor.app.officejs.com
# XXX Hard coded parameter
collector-db = /srv/slapgrid/var/data-log/collector.db
# Credentials
...
...
This diff is collapsed.
Click to expand it.
stack/monitor/scripts/monitor.py
View file @
ed459ed1
...
...
@@ -222,13 +222,18 @@ class Monitoring(object):
def
getMonitorTitleFromUrl
(
self
,
monitor_url
):
# This file should be generated
if
not
monitor_url
.
startswith
(
'https://'
)
and
not
monitor_url
.
startswith
(
'http://'
):
return
'Unknow Instance'
return
'Unknow
n
Instance'
if
not
monitor_url
.
endswith
(
'/'
):
monitor_url
=
monitor_url
+
'/'
context
=
ssl
.
_create_unverified_context
()
url
=
monitor_url
+
'/.jio_documents/monitor.global.json'
# XXX Hard Coded path
try
:
# XXX - working here with public url
if
hasattr
(
ssl
,
'_create_unverified_context'
):
context
=
ssl
.
_create_unverified_context
()
response
=
urllib2
.
urlopen
(
url
,
context
=
context
)
else
:
response
=
urllib2
.
urlopen
(
url
)
except
urllib2
.
HTTPError
:
return
'Unknow Instance'
else
:
...
...
This diff is collapsed.
Click to expand it.
Write
Preview
Markdown
is supported
0%
Try again
or
attach a new file
Attach a file
Cancel
You are about to add
0
people
to the discussion. Proceed with caution.
Finish editing this message first!
Cancel
Please
register
or
sign in
to comment