Basic setup.py / Makefile to build/install/sdist stuff + bigfile.so skeleton

It is an early project decision to use gnu99 & Plan9 C extensions, to
simplify C code.

So far we only build stub wendelin/bigfile/_bigfile.so .

Makefile is introduced because there will be targets which are easier to
handle at make level. For end users no make knowledge is required -
usual `python setup.py build|install|...` work, redirecting to make
where necessary.

As was promised (e870781d "Top-level in-tree import redirector")
setup.py contains install-time hooks to handle in-tree wendelin.py and
install it as a module namespace.

For sdist, we just use `git ls-files` info if we are in a checkout.

Makefile

+# Wendelin.core | Instructions to build & test
+# Copyright (C) 2014-2015  Nexedi SA and Contributors.
+#                     	   Kirill Smelkov <kirr@nexedi.com>
+#
+# This program is free software: you can Use, Study, Modify and Redistribute
+# it under the terms of the GNU General Public License version 3, or (at your
+# option) any later version, as published by the Free Software Foundation.
+#
+# You can also Link and Combine this program with other software covered by
+# the terms of any of the Open Source Initiative approved licenses and Convey
+# the resulting work. Corresponding source of such a combination shall include
+# the source code for all other software used.
+#
+# This program is distributed WITHOUT ANY WARRANTY; without even the implied
+# warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
+#
+# See COPYING file for full licensing terms.
+all	:
+
+PYTHON	?= python
+
+# use the same C compiler as python
+# (for example it could be `gcc -m64` on a 32bit userspace)
+CC	:= $(shell $(PYTHON) -c "from __future__ import print_function;	\
+		import sysconfig as _;	\
+		print(_.get_config_vars()['CC'])")
+ifeq ($(CC),)
+$(error "Cannot defermine py-CC")
+endif
+
+all	: bigfile/_bigfile.so
+
+
+bigfile/_bigfile.so : 3rdparty/ccan/config.h FORCE
+	$(PYTHON) setup.py ll_build_ext --inplace
+
+
+FORCE	:
+
+
+# TODO add FORCE?
+3rdparty/ccan/config.h: 3rdparty/ccan/Makefile
+	$(MAKE) -C $(@D) $(@F)
+
+# if there is no ccan/Makefile - ccan submodule has not been initialized
+# if there is - it is ok, as that Makefile does not need to be rebuilt
+3rdparty/ccan/Makefile:
+	@echo 'E: 3rdparty/ccan submodule not initialized'
+	@echo 'E: please do `git submodule update --init`'
+	@false

bigfile/_bigfile.c

+/* Wendelin.bigfile | Python interface to memory/files
+ * Copyright (C) 2014-2015  Nexedi SA and Contributors.
+ *                          Kirill Smelkov <kirr@nexedi.com>
+ *
+ * This program is free software: you can Use, Study, Modify and Redistribute
+ * it under the terms of the GNU General Public License version 3, or (at your
+ * option) any later version, as published by the Free Software Foundation.
+ *
+ * You can also Link and Combine this program with other software covered by
+ * the terms of any of the Open Source Initiative approved licenses and Convey
+ * the resulting work. Corresponding source of such a combination shall include
+ * the source code for all other software used.
+ *
+ * This program is distributed WITHOUT ANY WARRANTY; without even the implied
+ * warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
+ *
+ * See COPYING file for full licensing terms.
+ */
+
+#include <Python.h>
+
+
+static /*const*/ PyMethodDef pybigfile_modulemeths[] = {
+    {NULL}
+};
+
+
+/* module init */
+#if PY_MAJOR_VERSION >= 3
+static /*const*/ PyModuleDef pybigfile_moduledef = {
+    PyModuleDef_HEAD_INIT,
+    .m_name     = "_bigfile",
+    .m_size     = -1,   /* = disable creating several instances of this module */
+    .m_methods  = pybigfile_modulemeths,
+};
+#endif
+
+static PyObject *
+_init_bigfile(void)
+{
+    PyObject *m;
+
+#if PY_MAJOR_VERSION >= 3
+    m = PyModule_Create(&pybigfile_moduledef);
+#else
+    m = Py_InitModule("_bigfile", pybigfile_modulemeths);
+#endif
+    if (!m)
+        return NULL;
+
+    return m;
+}
+
+
+#if PY_MAJOR_VERSION < 3
+# define PyInit__bigfile init_bigfile
+#endif
+__attribute__((visibility("default")))  // XXX should be in PyMODINIT_FUNC
+PyMODINIT_FUNC
+PyInit__bigfile(void)
+{
+#if PY_MAJOR_VERSION >= 3
+    return
+#endif
+        _init_bigfile();
+}

setup.py

+# Wendelin.core | pythonic package setup
+# Copyright (C) 2014-2015  Nexedi SA and Contributors.
+#                          Kirill Smelkov <kirr@nexedi.com>
+#
+# This program is free software: you can Use, Study, Modify and Redistribute
+# it under the terms of the GNU General Public License version 3, or (at your
+# option) any later version, as published by the Free Software Foundation.
+#
+# You can also Link and Combine this program with other software covered by
+# the terms of any of the Open Source Initiative approved licenses and Convey
+# the resulting work. Corresponding source of such a combination shall include
+# the source code for all other software used.
+#
+# This program is distributed WITHOUT ANY WARRANTY; without even the implied
+# warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
+#
+# See COPYING file for full licensing terms.
+from setuptools import setup, Extension, Command, find_packages
+from setuptools.command.build_py import build_py as _build_py
+from setuptools.command.build_ext import build_ext as _build_ext
+from distutils.errors import DistutilsExecError
+from subprocess import check_output as runcmd
+
+import os
+
+
+_bigfile = Extension('wendelin.bigfile._bigfile',
+            sources = [
+                'bigfile/_bigfile.c',
+            ],
+            include_dirs = [
+                './include',
+                './3rdparty/ccan',
+                './3rdparty/include'
+            ],
+            define_macros       = [('_GNU_SOURCE',None)],
+            extra_compile_args  = [
+                '-std=gnu99',           # declarations inside for-loop
+                '-fplan9-extensions',   # anonymous-structs + simple inheritance
+                '-fvisibility=hidden',  # by default symbols not visible outside DSO
+            ],
+
+            # can't - at runtime links with either python (without libpython) or libpython
+            # linking with both libpython and python would be wrong
+            #extra_link_args     = [
+            #    '-Wl,--no-undefined',   # check DSO for undefined symbols at link time
+            #]
+            )
+
+
+# build_py that
+# - prevents in-tree wendelin.py & setup.py to be installed
+# - synthesizes wendelin/__init__.py on install
+class build_py(_build_py):
+
+    def find_package_modules(self, package, package_dir):
+        modules = _build_py.find_package_modules(self, package, package_dir)
+        try:
+            modules.remove(('wendelin', 'wendelin', 'wendelin.py'))
+            modules.remove(('wendelin', 'setup',    'setup.py'))
+        except ValueError:
+            pass    # was not there
+
+        return modules
+
+    def build_packages(self):
+        _build_py.build_packages(self)
+        # emit std namespaceing mantra to wendelin/__init__.py
+        self.initfile = self.get_module_outfile(self.build_lib, ('wendelin',), '__init__')
+        with open(self.initfile, 'w') as f:
+            f.write("# this is a namespace package (autogenerated)\n")
+            f.write("__import__('pkg_resources').declare_namespace(__name__)\n")
+
+
+    def get_outputs(self, include_bytecode=1):
+        outputs = _build_py.get_outputs(self, include_bytecode)
+
+        # add synthethized __init__.py to outputs, so that `pip uninstall`
+        # works without leaving it
+        outputs.append(self.initfile)
+        if include_bytecode:
+            if self.compile:
+                outputs.append(self.initfile + 'c')
+            if self.optimize:
+                outputs.append(self.initfile + 'o')
+
+        return outputs
+
+
+
+
+
+
+# run `make <target>`
+def runmake(target):
+    err = os.system('make %s' % target)
+    if err:
+        raise DistutilsExecError('Failed to execute `make %s`' % target)
+
+
+# create distutils command to "run `make <target>`"
+def viamake(target, help_text):
+    class run_viamake(Command):
+        user_options = []
+        def initialize_options(self):   pass
+        def finalize_options(self):     pass
+
+        description  = help_text
+        def run(self):
+            runmake(target)
+
+    return run_viamake
+
+
+# build_ext that
+# - builds via Makefile  (and thus pre-builds ccan)  XXX hacky
+class build_ext(_build_ext):
+
+    def run(self):
+        runmake('all')
+        return _build_ext.run(self)
+
+
+
+# register `func` as entry point `entryname` in `groupname` in distribution `distname` on the fly
+def register_as_entrypoint(func, entryname, groupname, distname):
+    from pkg_resources import working_set, EntryPoint
+    dist = working_set.by_key[distname]
+    entrypoint = EntryPoint(entryname, func.__module__, attrs=(func.__name__,),
+                                    extras=(), dist=dist)
+    group = dist.get_entry_map(groupname)
+    assert entryname not in group
+    group[entryname] = entrypoint
+
+def git_lsfiles(dirname):
+    # FIXME dirname is currently ignored
+    # XXX non-ascii names, etc
+    for _ in runcmd(['git', 'ls-files']).splitlines():
+        yield _.decode('utf8')  # XXX utf8 hardcoded
+    # XXX recursive submodules
+    for _ in runcmd(['git', 'submodule', 'foreach', '--quiet',  \
+                r'git ls-files | sed "s|\(.*\)\$|$path/\1|g"']).splitlines():
+        yield _.decode('utf8')  # XXX utf8 hardcoded
+
+# if we are in git checkout - inject git_lsfiles to setuptools.file_finders
+# entry-point on the fly.
+#
+# otherwise, for released sdist tarball we do not - that tarball already
+# contains generated SOURCES.txt and sdist, if run from unpacked tarball, would
+# reuse that information.
+#
+# (to setuptools - beause we have to use some distribution and we already
+#  depend on setuptools)
+if os.path.exists('.git'):  # FIXME won't work if we are checked out as e.g. submodule
+    register_as_entrypoint(git_lsfiles, 'git', 'setuptools.file_finders', 'setuptools')
+
+
+
+
+
+setup(
+    name        = 'wendelin.core',
+    description = 'Out-of-core NumPy arrays',
+    url         = 'http://www.wendelin.io/',
+    license     = 'GPLv3+ with wide exception for Open-Source',
+    author      = 'Kirill Smelkov',
+    author_email= 'kirr@nexedi.com',
+
+    keywords    = 'bigdata out-of-core numpy virtual-memory',
+
+    ext_modules = [_bigfile],
+
+    package_dir = {'wendelin': ''},
+    packages    = ['wendelin'] + ['wendelin.%s' % _ for _ in
+                        find_packages(exclude='3rdparty')],
+    cmdclass    = {'build_ext':     build_ext,
+                   'll_build_ext':  _build_ext, # original build_ext for Makefile
+                   'build_py':      build_py,
+                  },
+)