clammit: new software release

Clammit is an HTTP interface to the ClamAV virus scanner

component/clammit/buildout.cfg

+[buildout]
+extends =
+  ../git/buildout.cfg
+  ../golang/buildout.cfg
+
+parts =
+  clammit
+
+[clammit-repository]
+<= go-git-package
+go.importpath = github.com/ifad/clammit
+repository    = https://github.com/ifad/clammit.git
+revision      = v0.8.1
+
+[gowork]
+install =
+# We need a repository here, instead of a URL
+# (github.com/ifad/clammit@v0.8.1), as the module
+# definition in the go.mod is wrong (see
+# https://github.com/ifad/clammit/pull/38)
+    ${clammit-repository:location}:./...

software/clammit/buildout.hash.cfg

+# THIS IS NOT A BUILDOUT FILE, despite purposedly using a compatible syntax.
+# The only allowed lines here are (regexes):
+# - "^#" comments, copied verbatim
+# - "^[" section beginings, copied verbatim
+# - lines containing an "=" sign which must fit in the following categorie.
+#   - "^\s*filename\s*=\s*path\s*$" where "path" is relative to this file
+#     Copied verbatim.
+#   - "^\s*hashtype\s*=.*" where "hashtype" is one of the values supported
+#     by the re-generation script.
+#     Re-generated.
+# - other lines are copied verbatim
+# Substitution (${...:...}), extension ([buildout] extends = ...) and
+# section inheritance (< = ...) are NOT supported (but you should really
+# not need these here).
+
+[instance.cfg.in]
+filename = instance.cfg.in
+md5sum = a327c7aebd30df6c59c01078cc0efe3a
+
+[clamd.conf.in]
+filename = clamd.conf.in
+md5sum = ea04b5aab8ae5302d16227d5121af256
+
+[freshclam.conf.in]
+filename = freshclam.conf.in
+md5sum = fb87755b97e230e5d499c1d8fb9762a9
+
+[clammit.conf.in]
+filename = clammit.conf.in
+md5sum = b32336982401088d58b9a9938c37742f

software/clammit/clamd.conf.in

+Foreground yes
+LogFile {{ clamconfig["clamd-log-file"] }}
+LogRotate no
+LogTime yes
+PidFile {{ clamconfig["clamd-pid-file"] }}
+LocalSocket {{ clamconfig["clamd-socket-file"] }}
+DatabaseDirectory {{ clamconfig["clam-database-directory"] }}
+TemporaryDirectory {{ clamconfig["clamd-temporary-directory"] }}
+LocalSocketMode 660
+ReadTimeout 30
+# MaxScanTime is in milliseconds.
+MaxScanTime 60000

software/clammit/clammit.conf.in

+[ application ]
+listen            = {{ clamconfig["clammit-listen-address"] }}:{{ clamconfig["clammit-listen-port"] }}
+clamd-url         = {{ clamconfig["clamd-socket-file"] }}
+virus-status-code = 418
+log-file          = {{ clamconfig["clammit-log-file"] }}

software/clammit/freshclam.conf.in

+Foreground yes
+DatabaseDirectory {{ clamconfig["clam-database-directory"] }}
+UpdateLogFile {{ clamconfig["freshclam-log-file"] }}
+LogRotate no
+LogTime yes
+PidFile {{ clamconfig["freshclam-pid-file"] }}
+DatabaseMirror database.clamav.net
+NotifyClamd {{ clamd_config_file }}

software/clammit/instance.cfg.in

+{% import "caucase" as caucase with context %}
+
+[buildout]
+
+extends =
+  {{ template_monitor }}
+
+parts =
+  ${:clamd-parts}
+  ${:freshclam-parts}
+  ${:clammit-parts}
+  ${:frontend-parts}
+  monitor-base
+  publish-connection-parameter
+
+clamd-parts =
+  clamd
+  clamd-logrotate
+freshclam-parts =
+  freshclam
+  freshclam-cron
+  freshclam-logrotate
+clammit-parts =
+  clammit
+  clammit-port-listening-promise
+frontend-parts =
+  frontend
+  frontend-promise
+  caucased-promise
+  frontend-certificate-promise
+
+eggs-directory = {{ buildout['eggs-directory'] }}
+develop-eggs-directory = {{ buildout['develop-eggs-directory'] }}
+offline = true
+
+[instance-parameter]
+recipe = slapos.cookbook:slapconfiguration
+computer = ${slap-connection:computer-id}
+partition = ${slap-connection:partition-id}
+url = ${slap-connection:server-url}
+key = ${slap-connection:key-file}
+cert = ${slap-connection:cert-file}
+
+[publish-connection-parameter]
+recipe = slapos.cookbook:publish
+caucase-url = ${caucased:url}
+scan-url = ${frontend-config:url}/clammit/scan
+url = ${frontend-config:url}/clammit
+
+[directory]
+recipe = slapos.cookbook:mkdirectory
+home = ${buildout:directory}
+etc = ${:home}/etc
+etc.certificate = ${:etc}/certificate
+etc.promise = ${:etc}/promise
+etc.run = ${:etc}/run
+etc.service = ${:etc}/service
+srv = ${:home}/srv
+srv.backup.caucased = ${:srv}/backup/caucased
+srv.caucased = ${:srv}/caucased
+tmp = ${:home}/tmp
+var = ${:home}/var
+var.clamdb = ${:var}/clamdb
+var.log = ${:var}/log
+var.run = ${:var}/run
+
+[clam-config]
+clam-database-directory = ${directory:var.clamdb}
+clamd-log-file = ${directory:var.log}/clamd.log
+clamd-pid-file = ${directory:var.run}/clamd.pid
+clamd-socket-file = ${directory:var.run}/clamd.sock
+clamd-temporary-directory = ${directory:tmp}
+freshclam-log-file = ${directory:var.log}/freshclam.log
+freshclam-pid-file = ${directory:var.run}/freshclam.pid
+clammit-log-file = ${directory:var.log}/clammit.log
+clammit-listen-address = ${instance-parameter:ipv4-random}
+clammit-listen-port = 8438
+clammit-url = http://${:clammit-listen-address}:${:clammit-listen-port}
+
+################################################################################
+# Clamd
+################################################################################
+
+[clamd.conf]
+recipe = slapos.recipe.template:jinja2
+output = ${directory:etc}/clamd.conf
+url = {{ clamd_conf_path }}
+context =
+  section clamconfig clam-config
+
+[clamd]
+recipe = slapos.cookbook:wrapper
+command-line = {{ clamav.location }}/sbin/clamd --config-file ${clamd.conf:output}
+wrapper-path = ${directory:etc.service}/${:_buildout_section_name_}
+hash-files =
+  ${clamd.conf:output}
+hash-existing-files =
+  {{ buildout.directory }}/.completed
+
+[clamd-logrotate]
+<= logrotate-entry-base
+name = clamd
+log = ${clam-config:clamd-log-file}
+post = {{ buildout.directory }}/bin/slapos-kill --pidfile ${clam-config:clamd-pid-file} -s SIGHUP
+
+################################################################################
+# Freshclam
+################################################################################
+
+[freshclam.conf]
+recipe = slapos.recipe.template:jinja2
+output = ${directory:etc}/freshclam.conf
+url = {{ freshclam_conf_path }}
+context =
+  section clamconfig clam-config
+  key clamd_config_file clamd.conf:output
+
+[freshclam]
+recipe = slapos.cookbook:wrapper
+command-line = {{ clamav.location }}/bin/freshclam --config-file ${freshclam.conf:output}
+wrapper-path = ${directory:etc.run}/${:_buildout_section_name_}
+
+[freshclam-database-refresh-time]
+recipe = slapos.cookbook:random.time
+
+[freshclam-cron]
+recipe = slapos.cookbook:cron.d
+time = ${freshclam-database-refresh-time:time}
+cron-entries = ${cron:cron-entries}
+name = freshclam
+command = {{ clamav.location }}/bin/freshclam --config-file ${freshclam.conf:output}
+
+[freshclam-logrotate]
+<= logrotate-entry-base
+name = freshclam
+log = ${clam-config:freshclam-log-file}
+
+################################################################################
+# Clammit
+################################################################################
+
+[clammit.conf]
+recipe = slapos.recipe.template:jinja2
+output = ${directory:etc}/clammit.conf
+url = {{ clammit_conf_path }}
+context =
+  section clamconfig clam-config
+  key database_directory directory:var.clamdb
+
+[clammit]
+recipe = slapos.cookbook:wrapper
+command-line = {{ clammit_bin }} -config ${clammit.conf:output}
+wrapper-path = ${directory:etc.service}/${:_buildout_section_name_}
+hash-files =
+  ${clammit.conf:output}
+hash-existing-files =
+  {{ buildout.directory }}/.completed
+
+[clammit-port-listening-promise]
+recipe = slapos.cookbook:check_port_listening
+path = ${directory:etc.promise}/${:_buildout_section_name_}
+hostname= ${clam-config:clammit-listen-address}
+port = ${clam-config:clammit-listen-port}
+
+################################################################################
+# Caucase
+################################################################################
+
+[frontend-certificate-init-certificate]
+recipe = slapos.recipe.build
+init =
+  # pre-create a file at the path of the certificate,
+  # so that we can use hash-existing-files options
+  import pathlib
+  cert_file = pathlib.Path(self.buildout['frontend-certificate']['cert-file'])
+  if not cert_file.parent.exists():
+    cert_file.parent.mkdir(parents=True)
+  if not cert_file.exists():
+    cert_file.touch()
+
+[frontend-certificate]
+key-file = ${directory:etc.certificate}/${:_buildout_section_name_}.crt.key
+cert-file = ${directory:etc.certificate}/${:_buildout_section_name_}.crt
+common-name = ${:_buildout_section_name_}
+ca-file = ${directory:etc.certificate}/${:_buildout_section_name_}.ca.crt
+crl-file = ${directory:etc.certificate}/${:_buildout_section_name_}.crl
+init = ${frontend-certificate-init-certificate:init}
+
+{{
+caucase.updater(
+    prefix='frontend-certificate',
+    buildout_bin_directory=buildout['bin-directory'],
+    updater_path='${directory:etc.service}/frontend-certificate-updater',
+    url='${caucased:url}',
+    data_dir='${directory:srv}/caucase-updater',
+    crt_path='${frontend-certificate:cert-file}',
+    ca_path='${frontend-certificate:ca-file}',
+    crl_path='${frontend-certificate:crl-file}',
+    key_path='${frontend-certificate:key-file}',
+    template_csr='${frontend-certificate-prepare-csr:csr}',
+    openssl=openssl.location + "/bin",
+)}}
+
+[frontend-certificate-csr-config]
+recipe = slapos.recipe.template
+inline =
+  [req]
+  prompt = no
+  req_extensions = req_ext
+  distinguished_name = dn
+  [ dn ]
+  CN = frontend
+  [ req_ext ]
+  subjectAltName = @alt_names
+  [ alt_names ]
+  IP.1 = ${frontend-config:address}
+output = ${buildout:parts-directory}/${:_buildout_section_name_}/${:_buildout_section_name_}.txt
+
+[frontend-certificate-prepare-csr]
+recipe = plone.recipe.command
+command =
+  if [ ! -f '${:csr}' ] ; then
+   {{ openssl.location }}/bin/openssl req \
+      -newkey rsa \
+      -batch \
+      -new \
+      -sha256 \
+      -nodes \
+      -keyout /dev/null \
+      -config '${frontend-certificate-csr-config:output}' \
+      -out '${:csr}'
+  fi
+stop-on-error = true
+csr = ${directory:srv}/${:_buildout_section_name_}.csr.pem
+
+[caucased]
+port = 19980
+ip = ${instance-parameter:ipv6-random}
+netloc = [${:ip}]:${:port}
+url = http://${:netloc}/
+
+{{
+caucase.caucased(
+    prefix='caucased',
+    buildout_bin_directory=buildout['bin-directory'],
+    caucased_path='${directory:etc.service}/caucased',
+    backup_dir='${directory:srv.backup.caucased}',
+    data_dir='${directory:srv.caucased}',
+    netloc='${caucased:netloc}',
+    tmp='${directory:tmp}',
+    service_auto_approve_count=1,
+    user_auto_approve_count=0,
+    key_len=2048,
+)}}
+
+################################################################################
+# Local frontend
+################################################################################
+
+[frontend-config]
+address = ${instance-parameter:ipv6-random}
+port = 3000
+url = https://[${:address}]:${:port}
+socket = ${directory:var.run}/haproxy.sock
+user = admin
+password = ${frontend-password:passwd}
+
+[frontend-password]
+recipe = slapos.cookbook:generate.password
+username = admin
+
+[haproxy.conf]
+recipe = slapos.recipe.template:jinja2
+url = {{ haproxy_conf_path }}
+output = ${directory:etc}/${:_buildout_section_name_}
+context =
+  key pidfile :pidfile
+  key content :content
+content =
+  frontend listener
+    mode http
+    bind [${frontend-config:address}]:${frontend-config:port} ssl crt ${frontend-certificate:cert-file} alpn h2,http/1.1
+    default_backend servers
+  backend servers
+    server app ${clam-config:clammit-listen-address}:${clam-config:clammit-listen-port} check
+pidfile = ${directory:var.run}/haproxy.pid
+
+[frontend]
+recipe = slapos.cookbook:wrapper
+wrapper-path = ${directory:etc.service}/${:_buildout_section_name_}
+command-line =
+  {{ haproxy.location }}/sbin/haproxy -f ${haproxy.conf:output}
+hash-files =
+  ${haproxy.conf:output}
+  ${frontend-certificate:cert-file}
+
+[frontend-promise]
+recipe = slapos.cookbook:check_url_available
+path = ${directory:etc.promise}/${:_buildout_section_name_}.py
+url = ${frontend-config:url}/clammit/readyz
+dash_path = {{ dash.location }}/bin/dash
+curl_path = {{ curl.location }}/bin/curl

software/clammit/software.cfg

+[buildout]
+extends =
+  ../../component/clamav/buildout.cfg
+  ../../component/clammit/buildout.cfg
+  ../../component/curl/buildout.cfg
+  ../../component/dash/buildout.cfg
+  ../../stack/caucase/buildout.cfg
+  ../../stack/haproxy/default-backend.cfg
+  ../../stack/logrotate/buildout.cfg
+  ../../stack/slapos.cfg
+  buildout.hash.cfg
+
+parts =
+  slapos-cookbook
+  caucase-eggs
+  instance.cfg.in
+
+[instance.cfg.in]
+recipe = slapos.recipe.template:jinja2
+output = ${buildout:directory}/instance.cfg
+url = ${:_profile_base_location_}/${:filename}
+context =
+  section buildout buildout
+  section openssl openssl
+  section clamav clamav
+  section haproxy haproxy
+  section curl curl
+  section dash dash
+  raw clammit_bin ${gowork:bin}/clammit
+  key clamd_conf_path clamd.conf.in:target
+  key freshclam_conf_path freshclam.conf.in:target
+  key clammit_conf_path clammit.conf.in:target
+  key template_logrotate_base template-logrotate-base:output
+  key haproxy_conf_path stack-haproxy-default-backend-config:target
+  key template_monitor monitor2-template:output
+import-list =
+  file caucase caucase-jinja2-library:target
+
+[download-base]
+recipe = slapos.recipe.build:download
+url = ${:_profile_base_location_}/${:filename}
+
+[clamd.conf.in]
+<= download-base
+
+[freshclam.conf.in]
+<= download-base
+
+[clammit.conf.in]
+<= download-base

software/clammit/test/README.md

+Tests for the clammit Software Release

software/clammit/test/setup.py

+##############################################################################
+#
+# Copyright (c) 2024 Nexedi SA and Contributors. All Rights Reserved.
+#
+# WARNING: This program as such is intended to be used by professional
+# programmers who take the whole responsibility of assessing all potential
+# consequences resulting from its eventual inadequacies and bugs
+# End users who are looking for a ready-to-use solution with commercial
+# guarantees and support are strongly adviced to contract a Free Software
+# Service Company
+#
+# This program is Free Software; you can redistribute it and/or
+# modify it under the terms of the GNU General Public License
+# as published by the Free Software Foundation; either version 3
+# of the License, or (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA  02111-1307, USA.
+#
+##############################################################################
+from setuptools import setup, find_packages
+
+version = '0.1'
+name = 'slapos.test.clammit'
+long_description = open("README.md").read()
+
+setup(
+    name=name,
+    version=version,
+    description="Test for SlapOS' dufs",
+    long_description=long_description,
+    long_description_content_type='text/markdown',
+    maintainer="Nexedi",
+    maintainer_email="info@nexedi.com",
+    url="https://lab.nexedi.com/nexedi/slapos",
+    packages=find_packages(),
+    install_requires=[
+        'slapos.core',
+        'slapos.libnetworkcache',
+        'requests',
+    ],
+    zip_safe=True,
+    test_suite='test',
+)

software/clammit/test/test.py

+##############################################################################
+#
+# Copyright (c) 2024 Nexedi SA and Contributors. All Rights Reserved.
+#
+# WARNING: This program as such is intended to be used by professional
+# programmers who take the whole responsibility of assessing all potential
+# consequences resulting from its eventual inadequacies and bugs
+# End users who are looking for a ready-to-use solution with commercial
+# guarantees and support are strongly adviced to contract a Free Software
+# Service Company
+#
+# This program is Free Software; you can redistribute it and/or
+# modify it under the terms of the GNU General Public License
+# as published by the Free Software Foundation; either version 3
+# of the License, or (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA  02111-1307, USA.
+#
+##############################################################################
+
+import contextlib
+import io
+import os
+import pathlib
+import subprocess
+import tempfile
+import urllib.parse
+
+import requests
+
+from slapos.testing.testcase import makeModuleSetUpAndTestCaseClass
+
+setUpModule, SlapOSInstanceTestCase = makeModuleSetUpAndTestCaseClass(
+  os.path.abspath(
+    os.path.join(os.path.dirname(__file__), '..', 'software.cfg')))
+
+
+class ClammitTestCase(SlapOSInstanceTestCase):
+  def setUp(self):
+    self.connection_parameters = \
+        self.computer_partition.getConnectionParameterDict()
+    self.ca_cert = self._getCaucaseServiceCACertificate()
+
+  def _getCaucaseServiceCACertificate(self):
+    ca_cert = tempfile.NamedTemporaryFile(
+      prefix="ca.crt.pem",
+      mode="w",
+      delete=False,
+    )
+    ca_cert.write(
+      requests.get(
+        urllib.parse.urljoin(
+          self.connection_parameters['caucase-url'],
+          '/cas/crt/ca.crt.pem',
+        )).text)
+    self.addCleanup(os.unlink, ca_cert.name)
+    return ca_cert.name
+
+  def test_upload_of_files_to_clammit_for_scan(self):
+    resp = requests.get(
+      self.connection_parameters['scan-url'],
+      verify=self.ca_cert,
+    )
+
+    r = requests.post(
+      self.connection_parameters['scan-url'],
+      files={'file': 'Hello world'},
+      verify=self.ca_cert,
+    )
+    self.assertEqual(r.status_code, 200)
+
+    r = requests.post(
+      self.connection_parameters['scan-url'],
+      files={'file': b'X5O!P%@AP[4\PZX54(P^)7CC)7}$EICAR-STANDARD-ANTIVIRUS-TEST-FILE!$H+H*'},
+      verify=self.ca_cert,
+    )
+    self.assertEqual(r.status_code, 418)
+
+  def test_renew_certificate(self):
+    def _getpeercert():
+      # XXX low level way to get the server certificate
+      with requests.Session() as session:
+        pool = session.get(
+          self.connection_parameters['url'],
+          verify=self.ca_cert,
+        ).raw._pool.pool
+        with contextlib.closing(pool.get()) as cnx:
+          return cnx.sock._sslobj.getpeercert()
+
+    cert_before = _getpeercert()
+    # execute certificate updater when it's time to renew certificate.
+    # use a timeout, because this service runs forever
+    subprocess.run(
+      (
+        'timeout',
+        '5',
+        'faketime',
+        '+63 days',
+        os.path.join(
+          self.computer_partition_root_path,
+          'etc/service/frontend-certificate-updater'),
+      ),
+      capture_output=not self._debug,
+    )
+
+    # reprocess instance to get the new certificate, after removing the timestamp
+    # to force execution
+    (pathlib.Path(self.computer_partition_root_path) / '.timestamp').unlink()
+    self.waitForInstance()
+
+    cert_after = _getpeercert()
+    self.assertNotEqual(cert_before['notAfter'], cert_after['notAfter'])