Commit 31142ddf authored by Julien Muchembled's avatar Julien Muchembled

Use inotify-simple instead of inotifyx

There's little hope that it gets maintained because it contains a C extension whereas alternatives often use ctypes. In particular, it has no support for Python 3, and this is a blocker for us. At the beginning, [I though we were switching to pyinotify](0bb405fc): it is quite popular, and even used by fail2ban, but the code is ugly (big, crazy API, limited, and probably slow).

I didn't really know inotify and it's disappointing to see that created files (CREATE+WRITE+CLOSE_WRITE) can't be distinguished from hard links (CREATE). Acting upon new inodes is a common scenario and in the first case, you want to wait CLOSE_WRITE or you would read a partially written file. What I mean is that inotify is often unreliable, unless you detect changes done by your own software (e.g. you can make sure that files aren't hard-linked) but then some other IPC is probably simpler.

In any case, I open this MR because I haven't tested it. I only checked with pylint (hence the second commit).

/cc @alain.takoudjou @gabriel @luke (from Git history)

/reviewed-on !257
parents b184f687 607ec2c7
...@@ -55,11 +55,12 @@ setup(name=name, ...@@ -55,11 +55,12 @@ setup(name=name,
packages=find_packages(), packages=find_packages(),
include_package_data=True, include_package_data=True,
install_requires=[ install_requires=[
'enum34', # for inotify-simple
'jsonschema', 'jsonschema',
'hexagonit.recipe.download', 'hexagonit.recipe.download',
'netaddr', # to manipulate on IP addresses 'netaddr', # to manipulate on IP addresses
'setuptools', # namespaces 'setuptools', # namespaces
'inotifyx', # XXX use pyinotify instead 'inotify-simple',
'lock_file', #another lockfile implementation for multiprocess 'lock_file', #another lockfile implementation for multiprocess
'slapos.core', # uses internally 'slapos.core', # uses internally
'zc.buildout', # plays with buildout 'zc.buildout', # plays with buildout
......
...@@ -3,47 +3,31 @@ import os ...@@ -3,47 +3,31 @@ import os
import signal import signal
import subprocess import subprocess
import time import time
from collections import defaultdict
import inotifyx from inotify_simple import INotify, flags
def _wait_files_creation(file_list): def _wait_files_creation(file_list):
# Etablish a list of directory and subfiles # Establish a list of directory and subfiles.
directories = dict() # and test existence before watching, so that we don't miss an event.
for dirname, filename in [os.path.split(f) for f in file_list]: directories = defaultdict(dict)
directories.setdefault(dirname, dict()) for f in file_list:
directories[dirname][filename] = False dirname, filename = os.path.split(f)
directories[dirname][filename] = os.path.lexists(f)
def all_files_exists(): def all_files_exists():
return all([all(files.values()) for files in directories.values()]) return all(all(files.itervalues()) for files in directories.itervalues())
fd = inotifyx.init() with INotify() as inotify:
try: watchdescriptors = {inotify.add_watch(dirname,
# Watch every directories where the file are flags.CREATE | flags.DELETE | flags.MOVED_TO | flags.MOVED_FROM
watchdescriptors = dict() ): dirname
for dirname in directories.keys(): for dirname in directories}
wd = inotifyx.add_watch(fd,
dirname,
inotifyx.IN_CREATE | inotifyx.IN_DELETE | inotifyx.IN_MOVE)
watchdescriptors[wd] = dirname
# Set to True the file wich exists
for dirname, filename in [os.path.split(f) for f in file_list]:
directories[dirname][filename] = os.path.exists(os.path.join(dirname,
filename))
# Let's wait for every file creation
while not all_files_exists():
events_list = inotifyx.get_events(fd)
for event in events_list:
dirname = watchdescriptors[event.wd]
if event.name in directories[dirname]:
# One of watched file was created or deleted
if event.mask & inotifyx.IN_DELETE:
directories[dirname][event.name] = False
else:
directories[dirname][event.name] = True
finally: while not all_files_exists():
os.close(fd) for event in inotify.read():
directory = directories[watchdescriptors[event.wd]]
if event.name in directory:
directory[event.name] = event.mask & (flags.CREATE | flags.MOVED_TO)
def execute(args): def execute(args):
"""Portable execution with process replacement""" """Portable execution with process replacement"""
...@@ -83,8 +67,8 @@ def generic_exec(args): ...@@ -83,8 +67,8 @@ def generic_exec(args):
os.execve(exec_list[0], exec_list + sys.argv[1:], exec_env) os.execve(exec_list[0], exec_list + sys.argv[1:], exec_env)
def sig_handler(signal, frame): def sig_handler(sig, frame):
print 'Received signal %r, killing children and exiting' % signal print 'Received signal %r, killing children and exiting' % sig
if child_pg is not None: if child_pg is not None:
os.killpg(child_pg, signal.SIGHUP) os.killpg(child_pg, signal.SIGHUP)
os.killpg(child_pg, signal.SIGTERM) os.killpg(child_pg, signal.SIGTERM)
...@@ -97,6 +81,7 @@ signal.signal(signal.SIGTERM, sig_handler) ...@@ -97,6 +81,7 @@ signal.signal(signal.SIGTERM, sig_handler)
def execute_with_signal_translation(args): def execute_with_signal_translation(args):
"""Run process as children and translate from SIGTERM to another signal""" """Run process as children and translate from SIGTERM to another signal"""
global child_pg
child = subprocess.Popen(args, close_fds=True, preexec_fn=os.setsid) child = subprocess.Popen(args, close_fds=True, preexec_fn=os.setsid)
child_pg = child.pid child_pg = child.pid
try: try:
......
...@@ -25,8 +25,7 @@ ...@@ -25,8 +25,7 @@
# #
############################################################################## ##############################################################################
import os import os
from inotify_simple import INotify, flags
import inotifyx
def subfiles(directory): def subfiles(directory):
"""Return the list of subfiles of a directory, and wait for the newly created """Return the list of subfiles of a directory, and wait for the newly created
...@@ -34,18 +33,12 @@ def subfiles(directory): ...@@ -34,18 +33,12 @@ def subfiles(directory):
CAUTION : *DONT TRY TO CONVERT THE RESULT OF THIS FUNCTION INTO A LIST ! CAUTION : *DONT TRY TO CONVERT THE RESULT OF THIS FUNCTION INTO A LIST !
ALWAYS ITERATE OVER IT !!!*""" ALWAYS ITERATE OVER IT !!!*"""
watchfd = inotifyx.init()
inotifyx.add_watch(watchfd, directory, inotifyx.IN_CREATE)
try:
subfiles = set(os.listdir(directory)) with INotify() as inotify:
subfiles |= set([file_.name for file_ in inotifyx.get_events(watchfd, 0)]) inotify.add_watch(directory, flags.CLOSE_WRITE | flags.MOVED_TO)
names = os.listdir(directory)
while True: while True:
for file_ in subfiles: for name in names:
yield os.path.join(directory, file_) yield os.path.join(directory, name)
names = (event.name for event in inotify.read())
subfiles = [file_.name for file_ in inotifyx.get_events(watchfd)]
finally:
os.close(watchfd)
import os, shutil, tempfile, threading, unittest
from slapos.recipe.librecipe import execute, inotify
class TestInotify(unittest.TestCase):
def setUp(self):
self.tmp = tempfile.mkdtemp()
def tearDown(self):
shutil.rmtree(self.tmp)
def test_subfiles(self):
p = lambda x: os.path.join(self.tmp, x)
def create(name, text):
a = open(p(name), 'w')
a.write(text)
a.flush()
return a
def check(name, text):
path = next(notified)
self.assertEqual(path, p(name))
with open(path) as f:
self.assertEqual(f.read(), text)
a = create('first', 'blah')
a.write('...')
notified = inotify.subfiles(self.tmp)
check('first', 'blah')
os.link(p(a.name), p('a hard link')) # ignored
b = create('other', 'hello')
b.close()
check('other', 'hello')
c = create('last', '!!!')
a.close()
check('first', 'blah...')
os.rename(p(a.name), p(b.name))
check('other', 'blah...')
c.close()
check('last', '!!!')
def test_wait_files_creation(self):
file_list = (
'foo',
'bar',
'hello/world',
'hello/world!',
'a/b/c',
)
create = lambda x: open(x, 'w').close()
p = lambda x: os.path.join(self.tmp, x)
P = lambda x: p(file_list[x])
create(P(1))
os.mkdir(p('hello'))
os.makedirs(p('a/b'))
t = threading.Thread(target=execute._wait_files_creation,
args=(map(p, file_list),))
t.daemon = True
t.start()
def check():
t.join(.2)
self.assertTrue(t.is_alive())
check()
for x in P(3), p('a/b/d'), P(0):
create(x)
check()
os.rename(P(3), P(2))
os.rename(p('a/b/d'), P(4))
check()
os.remove(P(1))
for x in P(3), P(1):
create(x)
t.join(10)
self.assertFalse(t.is_alive())
...@@ -51,7 +51,6 @@ MarkupSafe = 0.18 ...@@ -51,7 +51,6 @@ MarkupSafe = 0.18
Werkzeug = 0.8.3 Werkzeug = 0.8.3
buildout-versions = 1.7 buildout-versions = 1.7
hexagonit.recipe.cmmi = 2.0 hexagonit.recipe.cmmi = 2.0
inotifyx = 0.2.0-1
lxml = 3.2.1 lxml = 3.2.1
meld3 = 0.6.10 meld3 = 0.6.10
netaddr = 0.7.10 netaddr = 0.7.10
......
...@@ -44,7 +44,6 @@ hexagonit.recipe.download = 1.6nxd002 ...@@ -44,7 +44,6 @@ hexagonit.recipe.download = 1.6nxd002
# Required by: # Required by:
# slapos.cookbook==0.73.1 # slapos.cookbook==0.73.1
inotifyx = 0.2.0
# Required by: # Required by:
# slapos.cookbook==0.73.1 # slapos.cookbook==0.73.1
......
...@@ -111,7 +111,7 @@ collective.recipe.template = 2.0 ...@@ -111,7 +111,7 @@ collective.recipe.template = 2.0
cryptography = 2.1.1 cryptography = 2.1.1
decorator = 4.0.11 decorator = 4.0.11
idna = 2.2 idna = 2.2
inotifyx = 0.2.2 inotify-simple = 1.1.1
itsdangerous = 0.24 itsdangerous = 0.24
lock-file = 2.0 lock-file = 2.0
lxml = 3.7.3 lxml = 3.7.3
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment