Merge pull request #503 from gotcha/centos-on-gh

Centos container on Github Actions

.dockerignore

+pythons
+python_builds
+download_cache
+parts
+eggs
+venvs
+.cache
+lib
+**/__pycache__
+**/*.pyc
+.git

.github/workflows/Dockerfile

+FROM centos as centos-python
+RUN yum install -y make gcc openssl-devel libffi-devel
+ARG PYTHON_VER
+ENV LC_CTYPE="en_US.UTF-8"
+RUN mkdir buildout
+WORKDIR /buildout
+COPY Makefile Makefile.configure /buildout/
+RUN make PYTHON_VER=${PYTHON_VER} download_python
+RUN make PYTHON_VER=${PYTHON_VER} python
+FROM centos-python
+ARG PYTHON_VER
+COPY doc /buildout/doc
+COPY src /buildout/src
+COPY zc.recipe.egg_ /buildout/zc.recipe.egg_
+COPY setup.* dev.py *.rst *.txt buildout.cfg /buildout/
+RUN make PYTHON_VER=${PYTHON_VER} build

.github/workflows/centos.yml

+name: Buildout on Centos
+
+on: [push]
+
+jobs:
+  build:
+
+    runs-on: ubuntu-latest
+    strategy:
+      fail-fast: false
+      matrix:
+        python-version: [3.5, 3.6, 3.7, 3.8]
+
+    steps:
+    - uses: actions/checkout@v2
+    - name: Build container with Python ${{ matrix.python-version }}
+      env:
+        PYTHON_VER: ${{matrix.python-version}}
+      run: |
+        docker build -f .github/workflows/Dockerfile --tag centos_buildout:python${PYTHON_VER} --build-arg PYTHON_VER=${PYTHON_VER} .
+    - name: Test
+      env:
+        PYTHON_VER: ${{matrix.python-version}}
+      run: |
+        docker run centos_buildout:python${PYTHON_VER} /buildout/bin/test -c -vvv

Makefile

 HERE = $(shell pwd)
-PYTHON_VER ?= 2.7
+PYTHON_VER ?= 3.7
 PYTHON_PATH = $(HERE)/pythons/$(PYTHON_VER)
 PYTHON_BUILD_DIR = $(HERE)/python_builds
+PLATFORM = $(shell uname)
+VENV = $(HERE)/venvs/$(PYTHON_VER)
+BUILD_VARIABLES =
 
 ifeq ($(PYTHON_VER),2.7)
-	PYTHON_MINOR ?= 2.7.13
-endif
-ifeq ($(PYTHON_VER),3.4)
-	PYTHON_MINOR ?= 3.4.6
-	PYTHON_CONFIGURE_ARGS ?= --without-ensurepip
+	PYTHON_MINOR ?= 2.7.17
 endif
 ifeq ($(PYTHON_VER),3.5)
-	PYTHON_MINOR ?= 3.5.3
-	PYTHON_CONFIGURE_ARGS ?= --without-ensurepip
+	PYTHON_MINOR ?= 3.5.9
 endif
 ifeq ($(PYTHON_VER),3.6)
-	PYTHON_MINOR ?= 3.6.0
-	PYTHON_CONFIGURE_ARGS ?= --without-ensurepip
+	PYTHON_MINOR ?= 3.6.10
+endif
+ifeq ($(PYTHON_VER),3.7)
+	PYTHON_MINOR ?= 3.7.7
+endif
+ifeq ($(PYTHON_VER),3.8)
+	PYTHON_MINOR ?= 3.8.2
+endif
+ifeq ($(PYTHON_VER),3.9)
+	PYTHON_MINOR ?= 3.9.0
+	PYTHON_ARCHIVE ?= Python-3.9.0a6
 endif
 
 ifndef PYTHON_MINOR
     $(error Please specify desired PYTHON_MINOR for Python $(PYTHON_VER))
 endif
 
+ifeq ($(PLATFORM),Darwin)
+	include Makefile.configure.Darwin
+else
+	include Makefile.configure
+endif
+
+
 PYTHON_ARCHIVE ?= Python-$(PYTHON_MINOR)
-PYTHON_DOWNLOAD = https://www.python.org/ftp/python/$(PYTHON_MINOR)/$(PYTHON_ARCHIVE).tgz
+PYTHON_URL = https://www.python.org/ftp/python/$(PYTHON_MINOR)/$(PYTHON_ARCHIVE).tgz
 PYTHON_EXE = python$(PYTHON_VER)
+PYTHON_DOWNLOAD = $(PYTHON_BUILD_DIR)/$(PYTHON_ARCHIVE).tgz
 
-.PHONY: all build test
 BUILD_DIRS = $(PYTHON_PATH) bin build develop-eggs eggs parts
 
-all: build
+all: test
+.PHONY: all download_python python build test docker clean
 
-$(PYTHON_PATH)/bin/$(PYTHON_EXE):
+# setup python from source
+$(PYTHON_DOWNLOAD):
+	mkdir -p $(PYTHON_BUILD_DIR)
+	curl --progress-bar --location $(PYTHON_URL) -o $(PYTHON_DOWNLOAD)
+
+$(PYTHON_BUILD_DIR)/$(PYTHON_ARCHIVE)/configure: $(PYTHON_DOWNLOAD)
+	cd $(PYTHON_BUILD_DIR) && tar -xzf $(PYTHON_DOWNLOAD)
+	touch $@
+
+$(PYTHON_PATH)/bin/$(PYTHON_EXE): $(PYTHON_BUILD_DIR)/$(PYTHON_ARCHIVE)/configure
 	@echo "Installing Python"
+	rm -rf $(PYTHON_PATH)
 	mkdir -p $(PYTHON_PATH)
-	mkdir -p $(PYTHON_BUILD_DIR)
-	cd $(PYTHON_BUILD_DIR) && \
-	curl --progress-bar --location $(PYTHON_DOWNLOAD) | tar -zx
 	cd $(PYTHON_BUILD_DIR)/$(PYTHON_ARCHIVE) && \
-	./configure --prefix $(PYTHON_PATH) $(PYTHON_CONFIGURE_ARGS) >/dev/null 2>&1 && \
+	$(BUILD_VARIABLES) ./configure --prefix $(PYTHON_PATH) $(PYTHON_CONFIGURE_ARGS) >/dev/null 2>&1 && \
 	make >/dev/null 2>&1 && \
 	make install >/dev/null 2>&1
 	@echo "Finished installing Python"
 
-build: $(PYTHON_PATH)/bin/$(PYTHON_EXE)
+download_python: $(PYTHON_BUILD_DIR)/$(PYTHON_ARCHIVE)/configure
+
+python: $(PYTHON_PATH)/bin/$(PYTHON_EXE)
+
+# used by Dockerfile
+build: python
 	$(PYTHON_PATH)/bin/$(PYTHON_EXE) dev.py
 
+docker:
+	docker build -f .github/workflows/Dockerfile --tag centos_buildout:python${PYTHON_VER} --build-arg PYTHON_VER=${PYTHON_VER} .
+	docker run centos_buildout:python${PYTHON_VER} /buildout/bin/test -c -vvv -t abi
+
 clean:
 	rm -rf $(BUILD_DIRS) $(PYTHON_BUILD_DIR)
 

Makefile.configure

+ifeq ($(PYTHON_VER),2.7)
+	PYTHON_CONFIGURE_ARGS ?=
+endif
+ifeq ($(PYTHON_VER),3.5)
+	PYTHON_CONFIGURE_ARGS ?= --without-ensurepip
+endif
+ifeq ($(PYTHON_VER),3.6)
+	PYTHON_CONFIGURE_ARGS ?= --without-ensurepip
+endif
+ifeq ($(PYTHON_VER),3.7)
+	PYTHON_CONFIGURE_ARGS ?= --without-ensurepip
+endif
+ifeq ($(PYTHON_VER),3.8)
+	PYTHON_CONFIGURE_ARGS ?= --without-ensurepip
+endif
+

Makefile.configure.Darwin

+OPENSSL = $(shell brew --prefix openssl)
+
+ifeq ($(PYTHON_VER),2.7)
+        BUILD_VARIABLES = LDFLAGS="-L$(OPENSSL)/lib" CPPFLAGS="-I$(OPENSSL)/include"
+	PYTHON_CONFIGURE_ARGS ?=
+endif
+ifeq ($(PYTHON_VER),3.5)
+        BUILD_VARIABLES = LDFLAGS="-L$(OPENSSL)/lib" CPPFLAGS="-I$(OPENSSL)/include"
+	PYTHON_CONFIGURE_ARGS ?= --without-ensurepip
+endif
+ifeq ($(PYTHON_VER),3.6)
+        BUILD_VARIABLES = LDFLAGS="-L$(OPENSSL)/lib" CPPFLAGS="-I$(OPENSSL)/include"
+	PYTHON_CONFIGURE_ARGS ?= --without-ensurepip
+endif
+ifeq ($(PYTHON_VER),3.7)
+	PYTHON_CONFIGURE_ARGS ?= --without-ensurepip --with-openssl=$(OPENSSL)
+endif
+ifeq ($(PYTHON_VER),3.8)
+	PYTHON_CONFIGURE_ARGS ?= --without-ensurepip --with-openssl=$(OPENSSL)
+endif
+ifeq ($(PYTHON_VER),3.9)
+	PYTHON_CONFIGURE_ARGS ?= --without-ensurepip --with-openssl=$(OPENSSL)
+endif
+

buildout.cfg

@@ -24,9 +24,10 @@ eggs =
   zc.recipe.egg
   coverage
 initialization =
-  import os
-  if 'COVERAGE_PROCESS_START' not in os.environ: os.environ['COVERAGE_PROCESS_START'] = os.path.abspath('../../.coveragerc')
-  if os.getenv('COVERAGE_PROCESS_START'): import coverage; coverage.process_startup()
+  import os.path
+  from zc.buildout.testing import setup_coverage
+  path_to_coveragerc = os.path.join('${buildout:directory}', '.coveragerc')
+  setup_coverage(path_to_coveragerc)
 
 # Tests that can be run wo a network
 [oltest]

src/zc/buildout/testing.py

@@ -297,15 +297,8 @@ def buildoutSetUp(test):
     # way due to the trick above:
     os.mkdir('develop-eggs')
 
-    if os.getenv("COVERAGE_PROCESS_START"):
-        # The user has requested subprocess code coverage. Since we will be changing
-        # directories, we need to make sure this path is absolute, which means
-        # we need to temporarily return to our starting directory.
-        os.chdir(here)
-        path_to_coveragerc = os.path.abspath(os.environ['COVERAGE_PROCESS_START'])
-        os.chdir(sample)
-        assert os.path.isfile(path_to_coveragerc), path_to_coveragerc
-        os.environ['COVERAGE_PROCESS_START'] = path_to_coveragerc
+    path_to_coveragerc = os.getenv("COVERAGE_PROCESS_START", None)
+    if path_to_coveragerc is not None:
 
         # Before we return to the current directory and destroy the
         # temporary working directory, we need to copy all the coverage files
@@ -608,6 +601,12 @@ ignore_not_upgrading = (
     'Not upgrading because not running a local buildout command.\n'
     ), '')
 
+easy_install_deprecated = (
+    re.compile(
+    'WARNING: The easy_install command is deprecated and will be removed in a future version.\n'
+    ), '')
+
+
 def run_buildout(command):
     # Make sure we don't get .buildout
     os.environ['HOME'] = os.path.join(os.getcwd(), 'home')
@@ -641,3 +640,41 @@ def run_buildout_in_process(command='buildout'):
         )
     command = ' '.join(command)
     run_in_process(run_buildout, command)
+
+
+def setup_coverage(path_to_coveragerc):
+    if 'RUN_COVERAGE' not in os.environ:
+        return
+    if not os.path.exists(path_to_coveragerc):
+        raise ValueError('coveragerc file %s does not exist.' % path_to_coveragerc)
+    os.environ['COVERAGE_PROCESS_START'] = path_to_coveragerc
+    rootdir = os.path.dirname(path_to_coveragerc)
+
+    def combine_report():
+        subprocess.call(
+            [
+                sys.executable, '-m', 'coverage', 'combine',
+            ],
+            cwd=rootdir,
+        )
+        subprocess.call(
+            [
+                sys.executable, '-m', 'coverage', 'report',
+            ],
+            cwd=rootdir,
+        )
+
+    if path_to_coveragerc:
+        try:
+            import coverage
+            print("Coverage configured with %s" % path_to_coveragerc)
+            if 'COVERAGE_REPORT' in os.environ:
+                import atexit
+                atexit.register(combine_report)
+            coverage.process_startup()
+        except ImportError:
+            print(
+                "You try to run coverage "
+                "but coverage is not installed in your environment."
+            )
+            sys.exit(1)

src/zc/buildout/tests.py

@@ -3600,6 +3600,7 @@ def test_suite():
                     zc.buildout.testing.normalize_script,
                     zc.buildout.testing.normalize_egg_py,
                     zc.buildout.testing.not_found,
+                    zc.buildout.testing.easy_install_deprecated,
                     zc.buildout.testing.adding_find_link,
                     # (re.compile(r"Installing 'zc.buildout >=\S+"), ''),
                     (re.compile(r'__buildout_signature__ = recipes-\S+'),
@@ -3643,6 +3644,7 @@ def test_suite():
                zc.buildout.testing.normalize_script,
                zc.buildout.testing.normalize_egg_py,
                zc.buildout.testing.not_found,
+               zc.buildout.testing.easy_install_deprecated,
                zc.buildout.testing.adding_find_link,
                # (re.compile(r"Installing 'zc.buildout >=\S+"), ''),
                # (re.compile(r"Getting distribution for 'zc.buildout >=\S+"),
@@ -3680,6 +3682,7 @@ def test_suite():
                 zc.buildout.testing.normalize_endings,
                 zc.buildout.testing.normalize_exception_type_for_python_2_and_3,
                 zc.buildout.testing.not_found,
+                zc.buildout.testing.easy_install_deprecated,
                 zc.buildout.testing.adding_find_link,
                 (re.compile('zc.buildout.buildout.MissingOption'),
                  'MissingOption'),
@@ -3703,6 +3706,7 @@ def test_suite():
                 zc.buildout.testing.normalize_script,
                 zc.buildout.testing.normalize_egg_py,
                 zc.buildout.testing.not_found,
+                zc.buildout.testing.easy_install_deprecated,
                 zc.buildout.testing.adding_find_link,
                 normalize_bang,
                 normalize_S,
@@ -3733,6 +3737,7 @@ def test_suite():
                 zc.buildout.testing.normalize_open_in_generated_script,
                 zc.buildout.testing.adding_find_link,
                 zc.buildout.testing.not_found,
+                zc.buildout.testing.easy_install_deprecated,
                 normalize_bang,
                 normalize_S,
                 (re.compile(r'[-d]  setuptools-\S+[.]egg'), 'setuptools.egg'),
@@ -3756,6 +3761,7 @@ def test_suite():
             checker=renormalizing.RENormalizing([
               zc.buildout.testing.normalize_exception_type_for_python_2_and_3,
               zc.buildout.testing.not_found,
+              zc.buildout.testing.easy_install_deprecated,
               zc.buildout.testing.adding_find_link,
               (re.compile(' at -?0x[^>]+'), '<MEM ADDRESS>'),
               (re.compile('http://localhost:[0-9]{4,5}/'),
@@ -3776,6 +3782,7 @@ def test_suite():
                 zc.buildout.testing.normalize_egg_py,
                 zc.buildout.testing.normalize___pycache__,
                 zc.buildout.testing.not_found,
+                zc.buildout.testing.easy_install_deprecated,
                 zc.buildout.testing.normalize_exception_type_for_python_2_and_3,
                 zc.buildout.testing.adding_find_link,
                 normalize_bang,
@@ -3824,6 +3831,7 @@ def test_suite():
                zc.buildout.testing.normalize_script,
                zc.buildout.testing.normalize_egg_py,
                zc.buildout.testing.not_found,
+               zc.buildout.testing.easy_install_deprecated,
                zc.buildout.testing.adding_find_link,
                (re.compile(r'__buildout_signature__ = recipes-\S+'),
                 '__buildout_signature__ = recipes-SSSSSSSSSSS'),
@@ -3904,6 +3912,7 @@ def test_suite():
                zc.buildout.testing.normalize_endings,
                zc.buildout.testing.normalize_script,
                zc.buildout.testing.not_found,
+               zc.buildout.testing.easy_install_deprecated,
                normalize_bang,
                zc.buildout.testing.adding_find_link,
                (re.compile('Downloading.*setuptools.*egg\n'), ''),

zc.recipe.egg_/src/zc/recipe/egg/tests.py

@@ -50,6 +50,7 @@ def test_suite():
                zc.buildout.tests.normalize_bang,
                zc.buildout.tests.normalize_S,
                zc.buildout.testing.not_found,
+               zc.buildout.testing.easy_install_deprecated,
                (re.compile(r'[d-]  zc.buildout(-\S+)?[.]egg(-link)?'),
                 'zc.buildout.egg'),
                (re.compile(r'[d-]  setuptools-[^-]+-'), 'setuptools-X-'),
@@ -65,6 +66,7 @@ def test_suite():
                zc.buildout.testing.normalize_path,
                zc.buildout.testing.normalize_endings,
                zc.buildout.testing.not_found,
+               zc.buildout.testing.easy_install_deprecated,
                (re.compile('__buildout_signature__ = '
                            r'sample-\S+\s+'
                            r'zc.recipe.egg-\S+\s+'
@@ -86,6 +88,7 @@ def test_suite():
                 zc.buildout.testing.normalize_path,
                 zc.buildout.testing.normalize_endings,
                 zc.buildout.testing.not_found,
+                zc.buildout.testing.easy_install_deprecated,
                 (re.compile("(d  ((ext)?demo(needed)?|other)"
                             r"-\d[.]\d-py)\d[.]\d(-\S+)?[.]egg"),
                  '\\1V.V.egg'),
@@ -117,6 +120,7 @@ def test_suite():
                zc.buildout.testing.normalize_path,
                zc.buildout.testing.normalize_endings,
                zc.buildout.testing.not_found,
+               zc.buildout.testing.easy_install_deprecated,
                ])
             ),
         ))