Commit fcd4e69f authored by Julien Muchembled's avatar Julien Muchembled

More optimization to wrapper scripts

The workaround for shebang size limitation is removed because it's redundant
with what is done by zc.buildout.easy_install and slapos.recipe.cmmi. This also
fix the issue that a process has a bad name even when the workaround is useless.
parent dfab94f0
...@@ -28,9 +28,37 @@ def _wait_files_creation(file_list): ...@@ -28,9 +28,37 @@ def _wait_files_creation(file_list):
if event.name in directory: if event.name in directory:
directory[event.name] = event.mask & (flags.CREATE | flags.MOVED_TO) directory[event.name] = event.mask & (flags.CREATE | flags.MOVED_TO)
def generic_exec(args, extra_environ=None, wait_list=None): def generic_exec(args, extra_environ=None, wait_list=None,
pidfile=None, reserve_cpu=False,
#shebang_workaround=False, # XXX: still needed ?
):
args = list(args)
if pidfile:
import psutil
try:
with open(pidfile) as f:
pid = int(f.read())
running = psutil.Process(pid).cmdline()
except Exception:
pass
else:
# With chained shebangs, several paths may be inserted at the beginning.
n = len(args)
for i in xrange(1+len(running)-n):
if args == running[i:n+i]:
sys.exit("Already running with pid %s." % pid)
with open(pidfile, 'w') as f:
f.write(str(os.getpid()))
args += sys.argv[1:] args += sys.argv[1:]
if reserve_cpu:
# If the CGROUPS cpuset is available (and prepared by slap format),
# request an exclusive CPU core for this process.
with open(os.path.expanduser('~/.slapos-cpu-exclusive'), 'a') as f:
f.write('%s\n' % os.getpid())
if wait_list: if wait_list:
_wait_files_creation(wait_list) _wait_files_creation(wait_list)
......
...@@ -33,7 +33,6 @@ import sys ...@@ -33,7 +33,6 @@ import sys
import inspect import inspect
import re import re
import shutil import shutil
from textwrap import dedent
import urllib import urllib
import urlparse import urlparse
...@@ -139,57 +138,25 @@ class GenericBaseRecipe(object): ...@@ -139,57 +138,25 @@ class GenericBaseRecipe(object):
def createWrapper(self, name, command, parameters, comments=(), def createWrapper(self, name, command, parameters, comments=(),
parameters_extra=False, environment=None, parameters_extra=False, environment=None,
pidfile=None, reserve_cpu=False
): ):
""" """
Creates a shell script for process replacement. Creates a basic shell script for process replacement.
Takes care of quoting. Takes care of quoting.
Takes care of #! line limitation when the wrapped command is a script.
if pidfile parameter is specified, then it will make the wrapper a singleton,
accepting to run only if no other instance is running.
:param reserve_cpu: bool, try to reserve one core for the `command` This must be kept minimal to avoid code duplication with generic_exec.
In particular, do not implement workaround for shebang size limitation here
(note that this can't be done correctly with a POSIX shell, because the
process can't be given a name).
""" """
lines = [ '#!/bin/sh' ] lines = [ '#!/bin/sh' ]
if comments: if comments:
lines += '# ', '\n# '.join(comments), '\n' lines += '# ', '\n# '.join(comments), '\n'
lines.append('COMMAND=' + shlex.quote(command))
for key in environment or (): for key in environment or ():
lines.append('export %s=%s' % (key, environment[key])) lines.append('export %s=%s' % (key, shlex.quote(environment[key])))
if pidfile: lines.append('exec ' + shlex.quote(command))
lines.append(dedent("""
# Check for other instances
pidfile=%s
if [ -s $pidfile ]; then
if pid=`pgrep -F $pidfile -f "$COMMAND" 2>/dev/null`; then
echo "Already running with pid $pid."
exit 1
fi
fi
echo $$ > $pidfile""" % shlex.quote(pidfile)))
if reserve_cpu:
# if the CGROUPS cpuset is available (and prepared by slap format)
# request an exclusive CPU core for this process
lines.append(dedent("""
# put own PID into waiting list for exclusive CPU-core access
echo $$ >> ~/.slapos-cpu-exclusive
"""))
lines.append(dedent('''
# If the wrapped command uses a shebang, execute the referenced
# executable passing the script path as first argument.
# This is to workaround the limitation of 127 characters in #!
[ ! -f "$COMMAND" ] || {
[ "`head -c2`" != "#!" ] || read -r EXE ARG
} < "$COMMAND"
exec $EXE ${ARG:+"$ARG"} "$COMMAND"'''))
parameters = map(shlex.quote, parameters) parameters = map(shlex.quote, parameters)
if parameters_extra: if parameters_extra:
......
...@@ -80,30 +80,34 @@ class Notify(GenericBaseRecipe): ...@@ -80,30 +80,34 @@ class Notify(GenericBaseRecipe):
# Just a touch # Just a touch
open(log, 'w').close() open(log, 'w').close()
parameters = [ cmd = [notifier_binary,
'-l', log, '-l', log,
'--title', title, '--title', title,
'--feed', feed_url, '--feed', feed_url,
'--max-run', str(max_run), '--max-run', str(max_run),
'--notification-url', '--notification-url',
] ]
parameters.extend(notification_url.split(' ')) cmd += notification_url.split(' ')
parameters.extend(['--executable', executable]) cmd += '--executable', executable
# For a more verbose mode, writing feed items for any action # For a more verbose mode, writing feed items for any action
instance_root_name = instance_root_name or self.options.get('instance-root-name', None) instance_root_name = instance_root_name or self.options.get('instance-root-name', None)
log_url = log_url or self.options.get('log-url', None) log_url = log_url or self.options.get('log-url', None)
status_item_directory = status_item_directory or self.options.get('status-item-directory', None) status_item_directory = status_item_directory or self.options.get('status-item-directory', None)
if instance_root_name and log_url and status_item_directory: if instance_root_name and log_url and status_item_directory:
parameters.extend([ cmd += (
'--instance-root-name', instance_root_name, '--instance-root-name', instance_root_name,
'--log-url', log_url, '--log-url', log_url,
'--status-item-directory', status_item_directory, '--status-item-directory', status_item_directory,
]) )
if pidfile:
return self.createPythonScript(wrapper,
'slapos.recipe.librecipe.execute.generic_exec',
(cmd,), {'pidfile': pidfile})
return self.createWrapper(name=wrapper, return self.createWrapper(name=wrapper,
command=notifier_binary, command=cmd[0],
parameters=parameters, parameters=cmd[1:],
pidfile=pidfile,
parameters_extra=True, parameters_extra=True,
comments=[ comments=[
'', '',
......
...@@ -26,7 +26,6 @@ ...@@ -26,7 +26,6 @@
############################################################################## ##############################################################################
import shlex import shlex
import os
from slapos.recipe.librecipe import GenericBaseRecipe from slapos.recipe.librecipe import GenericBaseRecipe
...@@ -45,54 +44,36 @@ class Recipe(GenericBaseRecipe): ...@@ -45,54 +44,36 @@ class Recipe(GenericBaseRecipe):
command_line = shlex.split(self.options['command-line']) command_line = shlex.split(self.options['command-line'])
wrapper_path = self.options['wrapper-path'] wrapper_path = self.options['wrapper-path']
wait_files = self.options.get('wait-for-files') wait_files = self.options.get('wait-for-files')
environment = self.options.get('environment')
parameters_extra = self.options.get('parameters-extra') parameters_extra = self.options.get('parameters-extra')
pidfile = self.options.get('pidfile') pidfile = self.options.get('pidfile')
reserve_cpu = self.options.get('reserve-cpu', False)
if not wait_files and not environment: environment = {}
# Create a simple wrapper as shell script for line in (self.options.get('environment') or '').splitlines():
return [self.createWrapper(
name=wrapper_path,
command=command_line[0],
parameters=command_line[1:],
parameters_extra=parameters_extra,
pidfile=pidfile,
reserve_cpu=reserve_cpu
)]
# More complex needs: create a Python script as wrapper
extra_environ = {}
if environment:
for line in environment.splitlines():
line = line.strip() line = line.strip()
if line: if line:
k, v = line.split('=') k, v = line.split('=')
extra_environ[k.rstrip()] = v.lstrip() environment[k.rstrip()] = v.lstrip()
args = [command_line, extra_environ] kw = {}
if wait_files: if wait_files:
args.append(wait_files.split()) kw['wait_list'] = wait_files.split()
if pidfile:
kw['pidfile'] = pidfile
if self.isTrueValue(self.options.get('reserve-cpu')):
kw['reserve_cpu'] = True
if kw:
# More complex needs: create a Python script as wrapper
args = [command_line]
if environment:
args.append(environment)
# We create a python script and a wrapper around the python return self.createPythonScript(wrapper_path,
# script because the python script might have a too long #! line
if os.path.exists(os.path.join(self.buildout['buildout']['directory'], "bin")):
base_script_path = os.path.join(
self.buildout['buildout']['directory'], "bin/" + wrapper_path.split("/")[-1])
else:
base_script_path = os.path.join(
self.buildout['buildout']['directory'], wrapper_path.split("/")[-1])
python_script = self.createPythonScript(
base_script_path +'.py',
'slapos.recipe.librecipe.execute.generic_exec', 'slapos.recipe.librecipe.execute.generic_exec',
args) args, kw)
return [python_script, self.createWrapper(
name=wrapper_path,
command=python_script,
parameters=[],
parameters_extra=parameters_extra,
pidfile=pidfile,
reserve_cpu=reserve_cpu
)]
return self.createWrapper(wrapper_path,
command_line[0],
command_line[1:],
parameters_extra=self.isTrueValue(parameters_extra),
environment=environment)
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment