Commit 2163fcaf authored by Kirill Smelkov's avatar Kirill Smelkov

wcfs: Initial stub

Add initial stub for WCFS program and tests.
WCFS functionality will be added step-by-step in follow-up commits.

Some preliminary history:

kirr/wendelin.core@0ae88a32       X .nxdtest: Verify Go bits with GOMAXPROCS=1,2,`nproc`
kirr/wendelin.core@23528eb4       X wcfs: make it to use go modules for dependencies
parent a05db040
...@@ -3,6 +3,10 @@ ...@@ -3,6 +3,10 @@
storv = ['fs', 'zeo', 'neo'] # storage backends to test against storv = ['fs', 'zeo', 'neo'] # storage backends to test against
# some bugs are only likely to trigger when there is only 1 or 2 main OS thread(s) in wcfs
# GOMAXPROCS='' means use `nproc`
gonprocv = ['1', '2', ''] # GOMAXPROCS=... to test against
# test.t & friends unit-test core of UVMM and are Go-, Python- and ZODB-storage independent. # test.t & friends unit-test core of UVMM and are Go-, Python- and ZODB-storage independent.
# we don't run test.vg* because there is currently no valgrind on SlapOS. # we don't run test.vg* because there is currently no valgrind on SlapOS.
...@@ -27,3 +31,17 @@ for stor in storv: ...@@ -27,3 +31,17 @@ for stor in storv:
TestCase('test.py/%s-!wcfs' % stor, ['make', 'test.py'], TestCase('test.py/%s-!wcfs' % stor, ['make', 'test.py'],
envadj={'WENDELIN_CORE_TEST_DB': '<%s>' % stor}, envadj={'WENDELIN_CORE_TEST_DB': '<%s>' % stor},
summaryf=PyTest.summary) summaryf=PyTest.summary)
# test.go unit-tests Go bits in wcfs.
for nproc in gonprocv:
TestCase('test.go:%s' % nproc, ['make', 'test.go'],
envadj={'GOMAXPROCS': nproc})
# test.wcfs/<stor> runs unit tests for WCFS
for stor in storv:
envdb = {'WENDELIN_CORE_TEST_DB': '<%s>' % stor}
for nproc in gonprocv:
TestCase('test.wcfs/%s:%s' % (stor, nproc), ['make', 'test.wcfs'],
envadj=dict(GOMAXPROCS=nproc, **envdb), summaryf=PyTest.summary)
...@@ -23,6 +23,7 @@ PYTHON ?= python ...@@ -23,6 +23,7 @@ PYTHON ?= python
PYTEST ?= $(PYTHON) -m pytest PYTEST ?= $(PYTHON) -m pytest
PYBENCH ?= $(PYTHON) -m golang.cmd.pybench PYBENCH ?= $(PYTHON) -m golang.cmd.pybench
VALGRIND?= valgrind VALGRIND?= valgrind
GO ?= go
# use the same C compiler as python # use the same C compiler as python
# (for example it could be `gcc -m64` on a 32bit userspace) # (for example it could be `gcc -m64` on a 32bit userspace)
...@@ -33,7 +34,7 @@ ifeq ($(CC),) ...@@ -33,7 +34,7 @@ ifeq ($(CC),)
$(error "Cannot defermine py-CC") $(error "Cannot defermine py-CC")
endif endif
all : bigfile/_bigfile.so all : bigfile/_bigfile.so wcfs/wcfs
ccan_config := 3rdparty/ccan/config.h ccan_config := 3rdparty/ccan/config.h
...@@ -42,6 +43,9 @@ bigfile/_bigfile.so : $(ccan_config) FORCE ...@@ -42,6 +43,9 @@ bigfile/_bigfile.so : $(ccan_config) FORCE
$(PYTHON) setup.py build_dso --inplace $(PYTHON) setup.py build_dso --inplace
$(PYTHON) setup.py ll_build_ext --inplace $(PYTHON) setup.py ll_build_ext --inplace
wcfs/wcfs: FORCE
cd wcfs && $(GO) build
FORCE : FORCE :
...@@ -69,7 +73,7 @@ CFLAGS := -g -Wall -D_GNU_SOURCE -std=gnu99 -fplan9-extensions \ ...@@ -69,7 +73,7 @@ CFLAGS := -g -Wall -D_GNU_SOURCE -std=gnu99 -fplan9-extensions \
# XXX hack ugly # XXX hack ugly
LOADLIBES=lib/bug.c lib/utils.c 3rdparty/ccan/ccan/tap/tap.c LOADLIBES=lib/bug.c lib/utils.c 3rdparty/ccan/ccan/tap/tap.c
TESTS := $(patsubst %.c,%,$(wildcard bigfile/tests/test_*.c)) TESTS := $(patsubst %.c,%,$(wildcard bigfile/tests/test_*.c))
test : test.t test.py test.fault test.asan test.tsan test.vgmem test.vghel test.vgdrd test : test.t test.go test.wcfs test.py test.fault test.asan test.tsan test.vgmem test.vghel test.vgdrd
# TODO move XFAIL markers into *.c # TODO move XFAIL markers into *.c
...@@ -165,17 +169,25 @@ test.vgdrd: $(TESTS:%=%.vgdrdrun) ...@@ -165,17 +169,25 @@ test.vgdrd: $(TESTS:%=%.vgdrdrun)
# run python tests # run python tests
PYTEST_IGNORE := --ignore=3rdparty --ignore=build --ignore=t PYTEST_IGNORE := --ignore=3rdparty --ignore=build --ignore=t
test.py : bigfile/_bigfile.so # wcfs unit-tests
$(PYTEST) $(PYTEST_IGNORE) test.wcfs : bigfile/_bigfile.so wcfs/wcfs
$(PYTEST) wcfs/
# unit/functional tests for whole wendelin.core
test.py : bigfile/_bigfile.so wcfs/wcfs
$(PYTEST) $(PYTEST_IGNORE) --ignore=wcfs
# test.py via Valgrind (very slow) # test.py via Valgrind (very slow)
test.py.vghel: bigfile/_bigfile.so test.py.vghel: bigfile/_bigfile.so wcfs/wcfs
$(call vgxrun,--tool=helgrind, $(PYTEST) $(PYTEST_IGNORE)) $(call vgxrun,--tool=helgrind, $(PYTEST) $(PYTEST_IGNORE))
test.py.drd: bigfile/_bigfile.so test.py.drd: bigfile/_bigfile.so wcfs/wcfs
$(call vgxrun,--tool=drd, $(PYTEST) $(PYTEST_IGNORE)) $(call vgxrun,--tool=drd, $(PYTEST) $(PYTEST_IGNORE))
# run go tests
test.go :
cd wcfs && $(GO) test -count=1 ./... # -count=1 disables tests caching
# test pagefault for double/real faults - it should crash # test pagefault for double/real faults - it should crash
tfault := bigfile/tests/tfault tfault := bigfile/tests/tfault
...@@ -194,5 +206,5 @@ bench : bench.t bench.py ...@@ -194,5 +206,5 @@ bench : bench.t bench.py
bench.t : $(BENCHV.C:%=%.trun) bench.t : $(BENCHV.C:%=%.trun)
bench.py: bigfile/_bigfile.so bench.py: bigfile/_bigfile.so wcfs/wcfs
$(PYBENCH) --count=3 --forked $(PYTEST_IGNORE) $(PYBENCH) --count=3 --forked $(PYTEST_IGNORE)
...@@ -21,11 +21,13 @@ from golang.pyx.build import setup, DSO as _DSO, Extension as _PyGoExt, build_ex ...@@ -21,11 +21,13 @@ from golang.pyx.build import setup, DSO as _DSO, Extension as _PyGoExt, build_ex
from setuptools import Command, find_packages from setuptools import Command, find_packages
from setuptools.command.build_py import build_py as _build_py from setuptools.command.build_py import build_py as _build_py
from pkg_resources import working_set, EntryPoint from pkg_resources import working_set, EntryPoint
from distutils.file_util import copy_file
from distutils.errors import DistutilsExecError from distutils.errors import DistutilsExecError
from subprocess import Popen, PIPE from subprocess import Popen, PIPE
import os import os
import sys import sys
from errno import EEXIST
# tell cython to resolve `cimport wendelin.*` modules hierarchy starting at top-level. # tell cython to resolve `cimport wendelin.*` modules hierarchy starting at top-level.
...@@ -172,11 +174,23 @@ def viamake(target, help_text): ...@@ -172,11 +174,23 @@ def viamake(target, help_text):
# build_ext that # build_ext that
# - builds via Makefile (and thus pre-builds ccan) XXX hacky # - builds via Makefile (and thus pre-builds ccan) XXX hacky
# - integrates built wcfs/wcfs.go exe into wcfs/ python package
class build_ext(_build_ext): class build_ext(_build_ext):
def run(self): def run(self):
runmake('all') runmake('all')
return _build_ext.run(self) _build_ext.run(self)
# copy wcfs/wcfs built from wcfs.go by make into build/lib.linux-x86_64-2.7/...
# so that py packaging tools see built wcfs.go as part of wendelin/wcfs package.
dst = os.path.join(self.build_lib, 'wendelin/wcfs')
try:
os.makedirs(dst)
except OSError as e:
if e.errno != EEXIST:
raise
copy_file('wcfs/wcfs', dst, verbose=self.verbose, dry_run=self.dry_run)
...@@ -329,6 +343,9 @@ setup( ...@@ -329,6 +343,9 @@ setup(
}, },
entry_points= {'console_scripts': [ entry_points= {'console_scripts': [
# start wcfs for zurl
'wcfs = wendelin.wcfs:main',
# demo to test # demo to test
'demo-zbigarray = wendelin.demo.demo_zbigarray:main', 'demo-zbigarray = wendelin.demo.demo_zbigarray:main',
] ]
......
/wcfs
*.test
# -*- coding: utf-8 -*-
# Copyright (C) 2018-2021 Nexedi SA and Contributors.
# Kirill Smelkov <kirr@nexedi.com>
#
# This program is free software: you can Use, Study, Modify and Redistribute
# it under the terms of the GNU General Public License version 3, or (at your
# option) any later version, as published by the Free Software Foundation.
#
# You can also Link and Combine this program with other software covered by
# the terms of any of the Free Software licenses or any of the Open Source
# Initiative approved licenses and Convey the resulting work. Corresponding
# source of such a combination shall include the source code for all other
# software used.
#
# This program is distributed WITHOUT ANY WARRANTY; without even the implied
# warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
#
# See COPYING file for full licensing terms.
# See https://www.nexedi.com/licensing for rationale and options.
"""Module wcfs.py provides python gateway for spawning wcfs server.
"""
from __future__ import print_function, absolute_import
import os, sys
from os.path import dirname
# _wcfs_exe returns path to wcfs executable.
def _wcfs_exe():
return '%s/wcfs' % dirname(__file__)
def main():
argv = sys.argv[1:]
os.execv(_wcfs_exe(), [_wcfs_exe()] + argv)
module lab.nexedi.com/nexedi/wendelin.core/wcfs
go 1.14
// Copyright (C) 2018-2021 Nexedi SA and Contributors.
// Kirill Smelkov <kirr@nexedi.com>
//
// This program is free software: you can Use, Study, Modify and Redistribute
// it under the terms of the GNU General Public License version 3, or (at your
// option) any later version, as published by the Free Software Foundation.
//
// You can also Link and Combine this program with other software covered by
// the terms of any of the Free Software licenses or any of the Open Source
// Initiative approved licenses and Convey the resulting work. Corresponding
// source of such a combination shall include the source code for all other
// software used.
//
// This program is distributed WITHOUT ANY WARRANTY; without even the implied
// warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
//
// See COPYING file for full licensing terms.
// See https://www.nexedi.com/licensing for rationale and options.
package main
func main() {
// XXX stub
}
# -*- coding: utf-8 -*-
# Copyright (C) 2018-2021 Nexedi SA and Contributors.
# Kirill Smelkov <kirr@nexedi.com>
#
# This program is free software: you can Use, Study, Modify and Redistribute
# it under the terms of the GNU General Public License version 3, or (at your
# option) any later version, as published by the Free Software Foundation.
#
# You can also Link and Combine this program with other software covered by
# the terms of any of the Free Software licenses or any of the Open Source
# Initiative approved licenses and Convey the resulting work. Corresponding
# source of such a combination shall include the source code for all other
# software used.
#
# This program is distributed WITHOUT ANY WARRANTY; without even the implied
# warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
#
# See COPYING file for full licensing terms.
# See https://www.nexedi.com/licensing for rationale and options.
"""wcfs_test.py tests wcfs filesystem from outside as python client process.
"""
from __future__ import print_function, absolute_import
from wendelin import wcfs
# TODO
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment