Commit ebb0421a authored by Jérome Perrin's avatar Jérome Perrin

new software: ProFTPd

Proftpd with sftp and virtual users

http://www.proftpd.org/docs/

# Features

 * sftp only is enabled
 * partially uploadloaded are not visible thanks to [`HiddenStores`](http://proftpd.org/docs/directives/linked/config_ref_HiddenStores.html) ( in fact they are, but name starts with `.` )
 * 5 failed login attempts will cause the host to be temporary banned


# TODO

 * only password login is enabled. enabling [`SFTPAuthorizedUserKeys`](http://www.proftpd.org/docs/contrib/mod_sftp.html#SFTPAuthorizedUserKeys) seems to break password only login
 * log rotation
 * make sure SFTPLog is useful (seems very verbose and does not contain more than stdout)
 * make it easier to manage users ( using `mod_auth_web` against an ERP5 endpoint or accepting a list of user/password as instance parameter )
 * allow configuring webhooks when new file is uploaded



----

depends on nexedi/slapos.core!41

/reviewed-on nexedi/slapos!311
parents b9347bd3 7683e129
# http://www.proftpd.org/ - Highly configurable GPL-licensed FTP server software
#
# Because it uses collective.recipe.environment this components needs slapos.core >= 1.4.7
# ( grid: do not hide `$USER` when running buildout )
#
[buildout]
extends =
../openssl/buildout.cfg
../perl/buildout.cfg
../zlib/buildout.cfg
../curl/buildout.cfg
../libtool/buildout.cfg
../git/buildout.cfg
# proftpd server
[proftpd-environment]
recipe = collective.recipe.environment
[proftpd-grp]
recipe = collective.recipe.grp
[proftpd]
recipe = slapos.recipe.cmmi
md5sum = 13270911c42aac842435f18205546a1b
url = ftp://ftp.proftpd.org/distrib/source/proftpd-1.3.6.tar.gz
configure-options =
--enable-openssl
--enable-nls
--enable-ctrls
--enable-dso
--with-modules=mod_sftp:mod_ban
--prefix=${buildout:parts-directory}/${:_buildout_section_name_}
environment =
CPPFLAGS=${:cppflags}
LDFLAGS=${:ldflags}
install_user=${proftpd-environment:USER}
install_group=${proftpd-grp:GROUP}
cppflags=-I${zlib:location}/include -I${openssl:location}/include
ldflags=-L${zlib:location}/lib -Wl,-rpath=${zlib:location}/lib -L${openssl:location}/lib -Wl,-rpath=${openssl:location}/lib
# mod_auth_web: a proftpd module to authenticate users against an HTTP service
[proftpd-mod_auth_web-repository]
recipe = slapos.recipe.build:gitclone
#repository = https://github.com/proftpd/mod_auth_web
# XXX until https://github.com/proftpd/mod_auth_web/pull/1 gets merged, we use
# the source repository of this PR directly
repository = https://github.com/jwm/mod_auth_web
revision = dec090bd0e287544a34be156ee17f715bd4286f9
git-executable = ${git:location}/bin/git
[proftpd-mod_auth_web]
recipe = plone.recipe.command
stop-on-error = true
command =
LIBTOOL=${libtool:location}/bin/libtool \
LDFLAGS="-Wl,-rpath=${curl:location}/lib ${proftpd:ldflags}" \
${proftpd-output:prxs} -c -i \
-I ${curl:location}/include ${proftpd:cppflags} \
-L ${curl:location}/lib/ \
-l curl \
-d ${proftpd-mod_auth_web-repository:location}/mod_auth_web.c
location=${proftpd:location}/libexec/mod_auth_web.so
# ftpasswod: a perl script to manage a proftpd AuthUserFile
[ftpasswd]
recipe = slapos.recipe.build:download
url = https://raw.githubusercontent.com/proftpd/proftpd/v1.3.6/contrib/ftpasswd
md5sum = 4a47df2cab86d8de7077a445bb416f31
download-only = true
mode = 0700
[proftpd-output]
# Shared binary location to ease migration
recipe = plone.recipe.command
stop-on-error = true
update-command = ${:command}
command = ${coreutils-output:test} -x ${:proftpd} -a -x ${:ftpasswd}
modules-deps =
${proftpd-mod_auth_web:recipe}
perl = ${perl:location}/bin/perl
ftpasswd = ${ftpasswd:target}
proftpd = ${proftpd:location}/sbin/proftpd
prxs = ${proftpd:location}/bin/prxs
ftpdctl = ${proftpd:location}/bin/ftpdctl
# ProFTPd test
This software release is simply to run the test suite from `../proftpd/test/setup.py`
Nexedi staff can see the results of this test from the test suite
`SLAPOS-PROFTPD-TEST` in test result module.
# THIS IS NOT A BUILDOUT FILE, despite purposedly using a compatible syntax.
# The only allowed lines here are (regexes):
# - "^#" comments, copied verbatim
# - "^[" section beginings, copied verbatim
# - lines containing an "=" sign which must fit in the following categorie.
# - "^\s*filename\s*=\s*path\s*$" where "path" is relative to this file
# But avoid directories, they are not portable.
# Copied verbatim.
# - "^\s*hashtype\s*=.*" where "hashtype" is one of the values supported
# by the re-generation script.
# Re-generated.
# - other lines are copied verbatim
# Substitution (${...:...}), extension ([buildout] extends = ...) and
# section inheritance (< = ...) are NOT supported (but you should really
# not need these here).
[template]
filename = instance.cfg
md5sum = 9b4cc727c43d0daaec5b01cf4b9f7a15
[buildout]
parts =
slapos-test-runner
eggs-directory = ${buildout:eggs-directory}
develop-eggs-directory = ${buildout:develop-eggs-directory}
offline = true
[slap-configuration]
recipe = slapos.cookbook:slapconfiguration
computer = $${slap-connection:computer-id}
partition = $${slap-connection:partition-id}
url = $${slap-connection:server-url}
key = $${slap-connection:key-file}
cert = $${slap-connection:cert-file}
[download-source]
recipe = slapos.recipe.build:gitclone
git-executable = ${git:location}/bin/git
[slapos]
<= download-source
repository = ${slapos-repository:location}
[create-directory]
recipe = slapos.cookbook:mkdirectory
bin = $${buildout:directory}/bin
[slapos-test-runner]
recipe = slapos.cookbook:wrapper
wrapper-path = $${create-directory:bin}/runTestSuite
command-line =
${buildout:bin-directory}/runTestSuite
--python_interpreter=${buildout:bin-directory}/${eggs:interpreter}
--source_code_path_list=$${slapos:location}/software/proftpd/test
# XXX slapos.cookbook:wrapper does not allow extending env, so we add some default $PATH entries ( not sure they are needed )
environment =
PATH=${buildout:bin-directory}:/usr/bin/:/bin/
LOCAL_IPV4=$${slap-configuration:ipv4-random}
GLOBAL_IPV6=$${slap-configuration:ipv6-random}
[buildout]
extends =
../../../../component/git/buildout.cfg
../../../../stack/slapos.cfg
./buildout.hash.cfg
parts =
slapos-cookbook
eggs
template
[setup-develop-egg]
recipe = zc.recipe.egg:develop
[slapos.test.proftpd-setup]
<= setup-develop-egg
egg = slapos.test.proftpd
setup = ${slapos-repository:location}/software/proftpd/test/
[erp5.util-setup]
<= setup-develop-egg
egg = erp5.util[testnode]
setup = ${erp5.util-repository:location}
[eggs]
recipe = zc.recipe.egg
eggs =
${slapos.test.proftpd-setup:egg}
${erp5.util-setup:egg}
slapos.core
entry-points =
runTestSuite=erp5.util.testsuite:runTestSuite
scripts =
runTestSuite
slapos
interpreter=
python_for_test
[git-clone-repository]
recipe = slapos.recipe.build:gitclone
git-executable = ${git:location}/bin/git
forbid-download-cache = true
branch = master
[slapos-repository]
<= git-clone-repository
repository = https://lab.nexedi.com/nexedi/slapos.git
# XXX we need an unreleased ( 0.4.51 ) version of erp5.util runTestSuite
# later we can stop fetching it from git and just use egg
[erp5.util-repository]
<= git-clone-repository
repository = https://lab.nexedi.com/nexedi/erp5.git
revision = 69013fa0fb67501089c776ab5e75d7bbf2e0e3bc
[template]
recipe = slapos.recipe.template
url = ${:_profile_base_location_}/${:filename}
output = ${buildout:directory}/template.cfg
mode = 640
[versions]
# clear the version of tested eggs, to make sure we installed the developped ones
slapos.test.proftpd =
erp5.util =
#erp5.util = 0.4.51
Proftpd with sftp and virtual users
http://www.proftpd.org/docs/
# Features
* sftp only is enabled
* partially uploadloaded are not visible thanks to [`HiddenStores`](http://proftpd.org/docs/directives/linked/config_ref_HiddenStores.html) ( in fact they are, but name starts with `.` )
* 5 failed login attempts will cause the host to be temporary banned
# TODO
* only password login is enabled. enabling [`SFTPAuthorizedUserKeys`](http://www.proftpd.org/docs/contrib/mod_sftp.html#SFTPAuthorizedUserKeys) seems to break password only login
* log rotation
* make sure SFTPLog is useful (seems very verbose and does not contain more than stdout)
* make it easier to manage users ( using `mod_auth_web` against an ERP5 endpoint or accepting a list of user/password as instance parameter )
* allow configuring webhooks when new file is uploaded
# THIS IS NOT A BUILDOUT FILE, despite purposedly using a compatible syntax.
# The only allowed lines here are (regexes):
# - "^#" comments, copied verbatim
# - "^[" section beginings, copied verbatim
# - lines containing an "=" sign which must fit in the following categorie.
# - "^\s*filename\s*=\s*path\s*$" where "path" is relative to this file
# But avoid directories, they are not portable.
# Copied verbatim.
# - "^\s*hashtype\s*=.*" where "hashtype" is one of the values supported
# by the re-generation script.
# Re-generated.
# - other lines are copied verbatim
# Substitution (${...:...}), extension ([buildout] extends = ...) and
# section inheritance (< = ...) are NOT supported (but you should really
# not need these here).
[instance-profile]
filename = instance.cfg.in
md5sum = 8ed5b4a7940db47ccb386c2f4e3e7273
[instance-default]
filename = instance-default.cfg.in
md5sum = 89e145188fe2a223f22f57e5c0c242e4
[proftpd-config-file]
filename = proftpd-config-file.cfg.in
md5sum = 46cbe4ea65b445822d3da32c76c7d67d
[buildout]
parts =
promises
publish-connection-parameter
[instance-parameter]
# TODO: this is not needed
recipe = slapos.cookbook:slapconfiguration
computer = ${slap-connection:computer-id}
partition = ${slap-connection:partition-id}
url = ${slap-connection:server-url}
key = ${slap-connection:key-file}
cert = ${slap-connection:cert-file}
[slap-configuration]
recipe = slapos.cookbook:slapconfiguration.serialised
computer = ${slap-connection:computer-id}
partition = ${slap-connection:partition-id}
url = ${slap-connection:server-url}
key = ${slap-connection:key-file}
cert = ${slap-connection:cert-file}
[directory]
recipe = slapos.cookbook:mkdirectory
home = ${buildout:directory}
etc = ${:home}/etc
var = ${:home}/var
log = ${:var}/log
srv = ${:home}/srv
service = ${:etc}/service
promise = ${:etc}/promise
proftpd-dir = ${:srv}/proftpd/
ssh-authorized-keys-dir = ${:etc}/authorized_keys/
[config-file]
recipe = slapos.recipe.template:jinja2
template = {{ software_parts_directory }}/${:_buildout_section_name_}/${:_buildout_section_name_}.cfg.in
rendered = ${directory:etc}/${:_buildout_section_name_}.cfg
mode = 0644
extensions = jinja2.ext.do
[check-port-listening-promise]
recipe = slapos.cookbook:check_port_listening
path = ${directory:promise}/${:_buildout_section_name_}
[proftpd-userinfo]
recipe = slapos.cookbook:userinfo
[proftpd-password]
recipe = slapos.cookbook:generate.password
username = proftpd
bytes = 12
[proftpd]
ipv6 = ${instance-parameter:ipv6-random}
ipv4 = ${instance-parameter:ipv4-random}
host = ${:ipv6}
sftp-port = {{ slapparameter_dict.get('port', 8022) }}
url = sftp://[${:host}]:${:sftp-port}
data-dir = ${directory:proftpd-dir}
user=${proftpd-userinfo:pw-name}
group=${proftpd-userinfo:gr-name}
sftp-log=${directory:log}/proftpd-sftp.log
xfer-log=${directory:log}/proftpd-xfer.log
ban-log=${directory:log}/proftpd-ban.log
ssh-host-rsa-key=${ssh-host-rsa-key:output}
ssh-host-dsa-key=${ssh-host-dsa-key:output}
ssh-host-ecdsa-key=${ssh-host-ecdsa-key:output}
ssh-authorized-keys-dir = ${directory:ssh-authorized-keys-dir}
ban-table=${directory:srv}/proftpd-ban-table
auth-user-file=${auth-user-file:output}
recipe = slapos.cookbook:wrapper
command-line =
{{ proftpd_bin }} --nodaemon --config ${proftpd-config-file:rendered}
wrapper-path = ${directory:service}/proftpd
[proftpd-listen-promise]
<= check-port-listening-promise
hostname = ${proftpd:ipv6}
port = ${proftpd:sftp-port}
[ftpasswd]
# command line to add a user, invoke with:
# ftpasswd --name=bob
# to prompt for password, or --stdin to read password from stdin
recipe = slapos.recipe.template:jinja2
mode = 0700
extensions = jinja2.ext.do
rendered = ${buildout:bin-directory}/${:_buildout_section_name_}
wrapper-path = ${:rendered}
# XXX: to workaround bug that pw-uid and gr-gir slapos.cookbook:userinfo cannot
# be used in a buildout part (as they are returned as int and not strings), we
# use slapos.recipe.template:jinja2 and not a simple slapos.cookbook:wrapper,
# because only the former is compatible with userinfo.
# Because we are already in a jinja template, we have to escape jinja template
# with {% raw %} / {% endraw %}
template = inline:
#!/bin/bash
{{ perl_bin }} {{ ftpasswd_bin }} --passwd --home=${proftpd:data-dir} --shell=/bin/false --uid={% raw %}{{ pw_uid }} --gid={{ gr_gid }}{% endraw %} --file ${auth-user-file:output} $@
context =
key pw_uid proftpd-userinfo:pw-uid
key gr_gid proftpd-userinfo:gr-gid
[auth-user-file]
recipe = plone.recipe.command
output = ${directory:etc}/ftpd.passwd
command =
echo ${proftpd-password:passwd} | ${ftpasswd:wrapper-path} --name=${proftpd-password:username} --stdin
update-command = ${:command}
[ssh-keygen-base]
recipe = plone.recipe.command
output = ${directory:etc}/${:_buildout_section_name_}
command = {{ ssh_keygen_bin }} -f ${:output} -N '' ${:extra-args}
[ssh-host-rsa-key]
<=ssh-keygen-base
extra-args=-t rsa
[ssh-host-dsa-key]
<=ssh-keygen-base
extra-args=-t dsa
[ssh-host-ecdsa-key]
<=ssh-keygen-base
extra-args=-t ecdsa -b 521
[proftpd-config-file]
<= config-file
context =
section proftpd proftpd
key slapparameter_dict slap-configuration:configuration
[promises]
recipe =
instance-promises =
${proftpd-listen-promise:path}
[publish-connection-parameter]
recipe = slapos.cookbook:publish
url = ${proftpd:url}
username = ${proftpd-password:username}
password = ${proftpd-password:passwd}
{
"$schema": "http://json-schema.org/draft-04/schema#",
"description": "Parameters to instantiate PoFTPd",
"additionalProperties": false,
"properties": {
"port": {
"description": "Port number to listen to - default to 8022",
"type": "number"
}
}
}
{
"$schema": "http://json-schema.org/draft-04/schema#",
"description": "Values returned by ProFTPd instantiation",
"additionalProperties": false,
"properties": {
"url": {
"description": "URL of the SFTP service",
"pattern": "^sftp://",
"type": "string"
},
"username": {
"description": "Default username",
"type": "string",
"optional": true
},
"password": {
"description": "Password for default username",
"type": "string",
"optional": true
}
},
"type": "object"
}
[buildout]
parts = switch-softwaretype
eggs-directory = {{ buildout['eggs-directory'] }}
develop-eggs-directory = {{ buildout['develop-eggs-directory'] }}
[slap-configuration]
recipe = slapos.cookbook:slapconfiguration.serialised
computer = ${slap-connection:computer-id}
partition = ${slap-connection:partition-id}
url = ${slap-connection:server-url}
key = ${slap-connection:key-file}
cert = ${slap-connection:cert-file}
[instance-template]
recipe = slapos.recipe.template:jinja2
filename = ${:_buildout_section_name_}.cfg
rendered = ${buildout:parts-directory}/${:_buildout_section_name_}/${:filename}
mode = 0644
extensions = jinja2.ext.do
context =
key slapparameter_dict slap-configuration:configuration
raw software_parts_directory {{ buildout['parts-directory'] }}
raw proftpd_bin {{ proftpd_bin }}
raw ftpasswd_bin {{ ftpasswd_bin }}
raw ftpdctl_bin {{ ftpdctl_bin }}
raw ssh_keygen_bin {{ ssh_keygen_bin }}
raw perl_bin {{ perl_bin }}
[instance-default]
<= instance-template
template = {{ instance_default }}
[switch-softwaretype]
recipe = slapos.cookbook:switch-softwaretype
# XXX Jerome: what is this override for ?
override = {{ dumps(override_switch_softwaretype |default) }}
default = instance-default:rendered
# BBB
RootSoftwareInstance = ${:default}
ServerName "ProFTPD SlapOS"
ServerType standalone
MaxInstances 30
DefaultAddress {{ proftpd['ipv6'] }} {{ proftpd['ipv4']}}
SocketBindTight on
DefaultServer off
Port {{ proftpd['sftp-port'] }}
User {{ proftpd['user'] }}
Group {{ proftpd['group'] }}
Umask 022
AllowOverwrite on
# SFTP
SFTPEngine on
SFTPHostKey {{ proftpd['ssh-host-rsa-key'] }}
SFTPHostKey {{ proftpd['ssh-host-dsa-key'] }}
SFTPHostKey {{ proftpd['ssh-host-ecdsa-key'] }}
#SFTPAuthorizedUserKeys file:{{ proftpd['ssh-authorized-keys-dir'] }}%u
# Logging
TransferLog {{ proftpd['xfer-log'] }}
SFTPLog {{ proftpd['sftp-log'] }}
BanLog {{ proftpd['ban-log'] }}
# Virtual users
RequireValidShell off
AuthUserFile {{ proftpd['auth-user-file'] }}
# Prevent partially uploaded files to be visible
HiddenStores on
DeleteAbortedStores on
AllowStoreRestart off
# Limitations of running as non-root user
WtmpLog off
# ( we cannot use DefaultRoot ~ )
# Prevent user for escaping their home
<Directory />
<Limit ALL>
DenyAll
</Limit>
</Directory>
<Directory {{ proftpd['data-dir'] }}>
<Limit ALL>
AllowAll
</Limit>
</Directory>
# Ban failed logins
MaxLoginAttempts 1
BanEngine on
BanTable {{ proftpd['ban-table'] }}
# 5 failed login attemps in 5 minutes -> ban for 20 minutes
BanOnEvent MaxLoginAttempts 5/00:05:00 00:20:00 "Too many Failed Login Attempts"
BanControlsACLs all allow user {{ proftpd['user'] }}
[buildout]
extends =
../../stack/slapos.cfg
../../component/openssh/buildout.cfg
../../component/proftpd/buildout.cfg
buildout.hash.cfg
versions = versions
parts =
slapos-cookbook
proftpd-config-file
instance-profile
[download-file-base]
recipe = slapos.recipe.build:download
url = ${:_profile_base_location_}/${:filename}
download-only = true
mode = 0644
[proftpd-config-file]
<= download-file-base
[instance-default]
<= download-file-base
[instance-profile]
recipe = slapos.recipe.template:jinja2
template = ${:_profile_base_location_}/${:filename}
rendered = ${buildout:directory}/instance.cfg
mode = 0644
extensions = jinja2.ext.do
context =
section buildout buildout
key instance_default instance-default:target
key proftpd_bin proftpd-output:proftpd
key ftpasswd_bin proftpd-output:ftpasswd
key ftpdctl_bin proftpd-output:ftpdctl
key ssh_keygen_bin openssh-output:keygen
key perl_bin proftpd-output:perl
[versions]
collective.recipe.environment = 1.1.0
collective.recipe.grp = 1.1.0
slapos.recipe.template = 4.3
plone.recipe.command = 1.1
{
"name": "ProFTPd",
"description": "ProFTPd as a SFTP server with virtual users",
"serialisation": "json-in-xml",
"software-type": {
"default": {
"title": "Default",
"description": "ProFTPd, with a default user and users added with command line",
"request": "instance-input-schema.json",
"response": "instance-output-schema.json",
"index": 0
}
}
}
Tests for ProFTPd software release
##############################################################################
#
# Copyright (c) 2018 Nexedi SA and Contributors. All Rights Reserved.
#
# WARNING: This program as such is intended to be used by professional
# programmers who take the whole responsibility of assessing all potential
# consequences resulting from its eventual inadequacies and bugs
# End users who are looking for a ready-to-use solution with commercial
# guarantees and support are strongly adviced to contract a Free Software
# Service Company
#
# This program is Free Software; you can redistribute it and/or
# modify it under the terms of the GNU General Public License
# as published by the Free Software Foundation; either version 3
# of the License, or (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program; if not, write to the Free Software
# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
#
##############################################################################
from setuptools import setup, find_packages
import glob
import os
version = '0.0.1.dev0'
name = 'slapos.test.proftpd'
long_description = open("README.md").read()
setup(name=name,
version=version,
description="Test for SlapOS' ProFTPd",
long_description=long_description,
long_description_content_type='text/markdown',
maintainer="Nexedi",
maintainer_email="info@nexedi.com",
url="https://lab.nexedi.com/nexedi/slapos",
packages=find_packages(),
install_requires=[
'slapos.core',
'erp5.util',
'pysftp',
],
zip_safe=True,
test_suite='test',
)
##############################################################################
#
# Copyright (c) 2018 Nexedi SA and Contributors. All Rights Reserved.
#
# WARNING: This program as such is intended to be used by professional
# programmers who take the whole responsibility of assessing all potential
# consequences resulting from its eventual inadequacies and bugs
# End users who are looking for a ready-to-use solution with commercial
# guarantees and support are strongly adviced to contract a Free Software
# Service Company
#
# This program is Free Software; you can redistribute it and/or
# modify it under the terms of the GNU General Public License
# as published by the Free Software Foundation; either version 3
# of the License, or (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program; if not, write to the Free Software
# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
#
##############################################################################
import os
import shutil
import urlparse
import tempfile
import StringIO
import subprocess
import pysftp
from paramiko.ssh_exception import SSHException
from paramiko.ssh_exception import AuthenticationException
import utils
# for development: debugging logs and install Ctrl+C handler
if os.environ.get('DEBUG'):
import logging
logging.basicConfig(level=logging.DEBUG)
import unittest
unittest.installHandler()
class ProFTPdTestCase(utils.SlapOSInstanceTestCase):
@classmethod
def getSoftwareURLList(cls):
return (os.path.abspath(os.path.join(os.path.dirname(__file__), '..', 'software.cfg')), )
def _getConnection(self, username=None, password=None, hostname=None):
"""Returns a pysftp connection connected to the SFTP
username and password can be specified and default to the ones from
instance connection parameters.
another hostname can also be passed.
"""
# this tells paramiko not to verify host key
cnopts = pysftp.CnOpts()
cnopts.hostkeys = None
parameter_dict = self.computer_partition.getConnectionParameterDict()
sftp_url = urlparse.urlparse(parameter_dict['url'])
return pysftp.Connection(
hostname or sftp_url.hostname,
port=sftp_url.port,
cnopts=cnopts,
username=username or parameter_dict['username'],
password=password or parameter_dict['password'])
class TestSFTPListen(ProFTPdTestCase):
def test_listen_on_ipv4(self):
self.assertTrue(self._getConnection(hostname=self.config['ipv4_address']))
def test_does_not_listen_on_all_ip(self):
with self.assertRaises(SSHException):
self._getConnection(hostname='0.0.0.0')
class TestSFTPOperations(ProFTPdTestCase):
"""Tests upload / download features we expect in SFTP server.
"""
def setUp(self):
self.upload_dir = os.path.join(
self.computer_partition_root_path, 'srv', 'proftpd')
def tearDown(self):
for name in os.listdir(self.upload_dir):
path = os.path.join(self.upload_dir, name)
if os.path.isfile(path) or os.path.islink(path):
os.remove(path)
else:
shutil.rmtree(path)
def test_simple_sftp_session(self):
with self._getConnection() as sftp:
# put a file
with tempfile.NamedTemporaryFile() as f:
f.write("Hello FTP !")
f.flush()
sftp.put(f.name, remotepath='testfile')
# it's visible in listdir()
self.assertEqual(['testfile'], sftp.listdir())
# and also in the server filesystem
self.assertEqual(['testfile'], os.listdir(self.upload_dir))
# download the file again, it should have same content
tempdir = tempfile.mkdtemp()
self.addCleanup(lambda : shutil.rmtree(tempdir))
local_file = os.path.join(tempdir, 'testfile')
retrieve_same_file = sftp.get('testfile', local_file)
with open(local_file) as f:
self.assertEqual(f.read(), "Hello FTP !")
def test_uploaded_file_not_visible_until_fully_uploaded(self):
test_self = self
class PartialFile(StringIO.StringIO):
def read(self, *args):
# file is not visible yet
test_self.assertNotIn('destination', os.listdir(test_self.upload_dir))
# it's just a hidden file
test_self.assertEqual(['.in.destination.'], os.listdir(test_self.upload_dir))
return StringIO.StringIO.read(self, *args)
with self._getConnection() as sftp:
sftp.sftp_client.putfo(PartialFile("content"), "destination")
# now file is visible
self.assertEqual(['destination'], os.listdir(self.upload_dir))
def test_partial_upload_are_deleted(self):
test_self = self
with self._getConnection() as sftp:
class ErrorFile(StringIO.StringIO):
def read(self, *args):
# at this point, file is already created on server
test_self.assertEqual(['.in.destination.'], os.listdir(test_self.upload_dir))
# simulate a connection closed
sftp.sftp_client.close()
return "something that will not be sent to server"
with self.assertRaises(IOError):
sftp.sftp_client.putfo(ErrorFile(), "destination")
# no half uploaded file is kept
self.assertEqual([], os.listdir(self.upload_dir))
def test_user_cannot_escape_home(self):
with self._getConnection() as sftp:
with self.assertRaisesRegexp(IOError, 'Permission denied'):
sftp.listdir('..')
with self.assertRaisesRegexp(IOError, 'Permission denied'):
sftp.listdir('/')
with self.assertRaisesRegexp(IOError, 'Permission denied'):
sftp.listdir('/tmp/')
class TestUserManagement(ProFTPdTestCase):
def test_user_can_be_added_from_script(self):
with self.assertRaisesRegexp(AuthenticationException, 'Authentication failed'):
self._getConnection(username='bob', password='secret')
subprocess.check_call(
'echo secret | %s/bin/ftpasswd --name=bob --stdin' % self.computer_partition_root_path,
shell=True)
self.assertTrue(self._getConnection(username='bob', password='secret'))
class TestBan(ProFTPdTestCase):
def test_client_are_banned_after_5_wrong_passwords(self):
# Simulate failed 5 login attempts
for i in range(5):
with self.assertRaisesRegexp(AuthenticationException, 'Authentication failed'):
self._getConnection(password='wrong')
# after that, even with a valid password we cannot connect
with self.assertRaisesRegexp(SSHException, 'Connection reset by peer'):
self._getConnection()
# ban event is logged
with open(os.path.join(
self.computer_partition_root_path, 'var', 'log', 'proftpd-ban.log')) as ban_log_file:
self.assertRegexpMatches(
ban_log_file.readlines()[-1],
'login from host .* denied due to host ban')
class TestInstanceParameterPort(ProFTPdTestCase):
@classmethod
def getInstanceParameterDict(cls):
cls.free_port = utils.findFreeTCPPort(cls.config['ipv4_address'])
return {'port': cls.free_port}
def test_instance_parameter_port(self):
parameter_dict = self.computer_partition.getConnectionParameterDict()
sftp_url = urlparse.urlparse(parameter_dict['url'])
self.assertEqual(self.free_port, sftp_url.port)
self.assertTrue(self._getConnection())
##############################################################################
#
# Copyright (c) 2018 Nexedi SA and Contributors. All Rights Reserved.
#
# WARNING: This program as such is intended to be used by professional
# programmers who take the whole responsibility of assessing all potential
# consequences resulting from its eventual inadequacies and bugs
# End users who are looking for a ready-to-use solution with commercial
# guarantees and support are strongly adviced to contract a Free Software
# Service Company
#
# This program is Free Software; you can redistribute it and/or
# modify it under the terms of the GNU General Public License
# as published by the Free Software Foundation; either version 3
# of the License, or (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program; if not, write to the Free Software
# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
#
##############################################################################
import unittest
import os
import socket
from contextlib import closing
import logging
from erp5.util.testnode.SlapOSControler import SlapOSControler
from erp5.util.testnode.ProcessManager import ProcessManager
import slapos
def findFreeTCPPort(ip=''):
"""Find a free TCP port to listen to.
"""
family = socket.AF_INET6 if ':' in ip else socket.AF_INET
with closing(socket.socket(family, socket.SOCK_STREAM)) as s:
s.bind((ip, 0))
return s.getsockname()[1]
class SlapOSInstanceTestCase(unittest.TestCase):
@classmethod
def getSoftwareURLList(cls):
"""Return URL of software releases to install.
To be defined by subclasses.
"""
raise NotImplementedError()
@classmethod
def getInstanceParameterDict(cls):
"""Return instance parameters
To be defined by subclasses if they need to request instance with specific
parameters.
"""
return {}
# TODO: allow subclasses to request a specific software type ?
@classmethod
def setUpClass(cls):
working_directory = os.environ.get(
'SLAPOS_TEST_WORKING_DIR',
os.path.join(os.path.dirname(__file__), '.slapos'))
# To prevent error: Cannot open an HTTP server: socket.error reported
# AF_UNIX path too long This `working_directory` should not be too deep.
# Socket path is 108 char max on linux
# https://github.com/torvalds/linux/blob/3848ec5/net/unix/af_unix.c#L234-L238
if len(working_directory + '/inst/supervisord.socket') > 108:
raise RuntimeError('working directory too deep, try setting SLAPOS_TEST_WORKING_DIR')
if not os.path.exists(working_directory):
os.mkdir(working_directory)
cls.config = config = {
"working_directory": working_directory,
"slapos_directory": working_directory,
"log_directory": working_directory,
"computer_id": 'slapos.test', # XXX
'proxy_database': os.path.join(working_directory, 'proxy.db'),
'partition_reference': cls.__name__,
# "proper" slapos command must be in $PATH
'slapos_binary': 'slapos',
}
# Some tests are expecting that local IP is not set to 127.0.0.1
ipv4_address = os.environ.get('LOCAL_IPV4', '127.0.1.1')
ipv6_address = os.environ['GLOBAL_IPV6']
config['proxy_host'] = config['ipv4_address'] = ipv4_address
config['ipv6_address'] = ipv6_address
config['proxy_port'] = findFreeTCPPort(ipv4_address)
config['master_url'] = 'http://{proxy_host}:{proxy_port}'.format(**config)
cls._process_manager = process_manager = ProcessManager()
# XXX this code is copied from testnode code
slapos_controler = SlapOSControler(
working_directory,
config
)
slapproxy_log = os.path.join(config['log_directory'], 'slapproxy.log')
logger = logging.getLogger(__name__)
logger.debug('Configured slapproxy log to %r', slapproxy_log)
software_url_list = cls.getSoftwareURLList()
slapos_controler.initializeSlapOSControler(
slapproxy_log=slapproxy_log,
process_manager=process_manager,
reset_software=False,
software_path_list=software_url_list)
process_manager.supervisord_pid_file = os.path.join(
slapos_controler.instance_root, 'var', 'run', 'supervisord.pid')
software_status_dict = slapos_controler.runSoftwareRelease(config, environment=os.environ)
# TODO: log more details in this case
assert software_status_dict['status_code'] == 0
instance_parameter_dict = cls.getInstanceParmeterDict()
instance_status_dict = slapos_controler.runComputerPartition(
config,
cluster_configuration=instance_parameter_dict,
environment=os.environ)
# TODO: log more details in this case
assert instance_status_dict['status_code'] == 0
# FIXME: similar to test node, only one (root) partition is really supported for now.
computer_partition_list = []
for i in range(len(software_url_list)):
computer_partition_list.append(
slapos_controler.slap.registerOpenOrder().request(
software_url_list[i],
# This is how testnode's SlapOSControler name created partitions
partition_reference='testing partition {i}'.format(i=i, **config),
partition_parameter_kw=instance_parameter_dict))
# expose some class attributes so that tests can use them:
# the ComputerPartition instances, to getInstanceParmeterDict
cls.computer_partition = computer_partition_list[0]
# the path of the instance on the filesystem, for low level inspection
cls.computer_partition_root_path = os.path.join(
config['working_directory'],
'inst',
cls.computer_partition.getId())
@classmethod
def tearDownClass(cls):
# FIXME: if setUpClass fail, this is not called and leaks zombie processes
cls._process_manager.killPreviousRun()
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment