slapos cli: add slapos cachelookup commands

This commit remove the following 2 commands:
 * slapos cache lookup
 * slapos cache source
And introduces the following 3 commands:
 * slapos cachelookup binary-sr
 * slapos cachelookup url
 * slapos cachelookup pypi

This is useful to check if some objects are in shacache for buildout.
Please note that the commands are using the latest keys (introduced in
nexedi/slapos.buildout!26)

See merge request nexedi/slapos.core!376

CHANGES.rst

 Changes
 =======
 
+1.7.6 (unreleased yet)
+------------------
+ * cachelookup: introduce slapos cachelookup {url, binary-sr, pypi} commands
+ * cache: remove "slapos cache {lookup, source}" commands
+
 1.7.5 (2022-03-21)
 ------------------
  * slapgrid: fix invocation of bootstrapBuildout

documentation/source/slapos.usage.rst

@@ -407,28 +407,29 @@ Go to the SlapOS Master web page, click ``Account``, then ``Token``.
 A token is valid for a single ``configure client`` command and will expire after one day.
 
 
-cache lookup
-~~~~~~~~~~~~
+cachelookup binary-sr
+~~~~~~~~~~~~~~~~~~~~~
 
-.. program-output:: python slapos help cache lookup
+.. program-output:: python slapos help cachelookup binary-sr
 
 
 Examples
 
-* See if the wordpress Software Release is available in precompiled format for our distribution::
-
-    $ slapos cache lookup http://git.erp5.org/gitweb/slapos.git/blob_plain/refs/tags/slapos-0.156:/software/kvm/software.cfg
-    Software URL: http://git.erp5.org/gitweb/slapos.git/blob_plain/refs/tags/slapos-0.156:/software/kvm/software.cfg
-    MD5:          4410088e11f370503e9d78db4cfa4ec4
-    -------------
-    Available for:
-    distribution     |   version    |       id       | compatible?
-    -----------------+--------------+----------------+-------------
-    CentOS           |          6.3 |     Final      | no
-    Fedora           |           17 | Beefy Miracle  | no
-    Ubuntu           |        12.04 |    precise     | yes
-    debian           |        6.0.6 |                | no
-    debian           |          7.0 |                | no
+* See if the KVM Software Release is available in precompiled format for our distribution::
+
+    $ slapos cachelookup binary-sr https://lab.nexedi.com/nexedi/slapos/raw/1.0.232/software/kvm/software.cfg
+    Software URL: https://lab.nexedi.com/nexedi/slapos/raw/1.0.232/software/kvm/software.cfg
+    MD5:          248e006f4d007ca7060b16c9b0cb6bc8
+    --------------------------------------------------------------------
+        multiarch     distribution version   id   compatible? verified? 
+    --------------------------------------------------------------------
+     x86_64-linux-gnu    Ubuntu     18.04  bionic      no        yes    
+     x86_64-linux-gnu    Ubuntu     20.04  focal       no        yes    
+     x86_64-linux-gnu    debian      10.9              no        yes    
+     x86_64-linux-gnu    debian      11.2             yes        yes    
+     x86_64-linux-gnu    debian      8.11              no        yes    
+     x86_64-linux-gnu    debian      9.13              no        yes    
+    --------------------------------------------------------------------
 
 You can also use the corresponding hash value in place of the URL.
 

setup.py

@@ -96,8 +96,9 @@ setup(name=name,
         ],
         'slapos.cli': [
           # Utilities
-          'cache lookup = slapos.cli.cache:CacheLookupCommand',
-          'cache source = slapos.cli.cache_source:CacheLookupCommand',
+          'cachelookup binary-sr = slapos.cli.cache_binarysr:CacheLookupCommand',
+          'cachelookup url = slapos.cli.cache_url:CacheLookupCommand',
+          'cachelookup pypi = slapos.cli.cache_pypi:CacheLookupCommand',
           'complete = slapos.cli.complete:CompleteCommand',
           # SlapOS Node commands
           'node bang = slapos.cli.bang:BangCommand',

slapos/cli/cache.py → slapos/cli/cache_binarysr.py

@@ -38,8 +38,6 @@ from slapos.grid import networkcache
 from slapos.cli.config import ConfigCommand
 from slapos.util import str2bytes
 
-FAILURE_EXIT_CODE = 10
-
 class CacheLookupCommand(ConfigCommand):
     """
     perform a query to the networkcache
@@ -50,6 +48,7 @@ class CacheLookupCommand(ConfigCommand):
     cache of the software release, and which ones are compatible
     with the OS you are currently running.
     """
+    command_group = 'cachelookup'
 
     def get_parser(self, prog_name):
         ap = super(CacheLookupCommand, self).get_parser(prog_name)
@@ -88,11 +87,11 @@ def do_lookup(logger, cache_dir, cache_url, signature_certificate_list,
         md5 = hashlib.md5(str2bytes(software_url)).hexdigest()
     try:
         entries = networkcache.download_entry_list(cache_url, cache_dir,
-            md5, logger, signature_certificate_list, software_url)
+            md5, logger, signature_certificate_list)
     except Exception:
         logger.critical('Error while looking object %s', software_url,
             exc_info=True)
-        return FAILURE_EXIT_CODE
+        return 1
 
     if not entries:
         logger.info('Object found in cache, but has no binary entries.')

slapos/cli/cache_pypi.py

+# -*- coding: utf-8 -*-
+##############################################################################
+#
+# Copyright (c) 2010-2014 Vifib SARL and Contributors.
+# All Rights Reserved.
+#
+# WARNING: This program as such is intended to be used by professional
+# programmers who take the whole responsibility of assessing all potential
+# consequences resulting from its eventual inadequacies and bugs
+# End users who are looking for a ready-to-use solution with commercial
+# guarantees and support are strongly adviced to contract a Free Software
+# Service Company
+#
+# This program is Free Software; you can redistribute it and/or
+# modify it under the terms of the GNU Lesser General Public License
+# as published by the Free Software Foundation; either version 2.1
+# of the License, or (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU Lesser General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA  02111-1307, USA.
+#
+##############################################################################
+
+import ast
+import hashlib
+import json
+import re
+import requests
+import sys
+import prettytable
+
+from six.moves.urllib.error import HTTPError
+
+from slapos.grid import networkcache
+from slapos.cli.config import ConfigCommand
+from slapos.cli.command import resetLogger
+from slapos.util import str2bytes
+
+class CacheLookupCommand(ConfigCommand):
+    """
+    perform a query to the networkcache.
+
+    Check if python package is available to be downloaded from cache.
+    """
+    command_group = 'cachelookup'
+
+    def get_parser(self, prog_name):
+        ap = super(CacheLookupCommand, self).get_parser(prog_name)
+        ap.add_argument('name',
+                        help='python package name')
+        ap.add_argument('version',
+                        help='python package version')
+        return ap
+
+    def take_action(self, args):
+        configp = self.fetch_config(args)
+        cache_dir = configp.get('networkcache', 'download-dir-url')
+        cache_url = configp.get('networkcache', 'download-cache-url')
+        signature_certificate_list = configp.get('networkcache', 'signature-certificate-list')
+
+        sys.exit(
+          do_lookup(self.app.log, cache_dir, cache_url,
+                    signature_certificate_list, args.name, args.version))
+
+def do_lookup(logger, cache_dir, cache_url, signature_certificate_list,
+              name, version):
+    key = 'pypi:{}={}'.format(name, version)
+    try:
+        entries = networkcache.download_entry_list(cache_url, cache_dir,
+            key, logger, signature_certificate_list)
+        if not entries:
+            logger.info('Object found in cache, but has no entry.')
+            return 0
+
+        pt = prettytable.PrettyTable(['basename', 'sha512', 'signed'])
+        for entry in entries:
+            d = json.loads(entry[0])
+            pt.add_row([d["basename"], d["sha512"], entry[1]])
+
+        logger.info('Python egg %s version %s', name, version)
+        logger.info('SHADIR URL: %s/%s\n', cache_dir, key)
+
+        resetLogger(logger)
+        for line in pt.get_string(border=True, padding_width=0, vrules=prettytable.NONE).split('\n'):
+            logger.info(line)
+    except HTTPError as e:
+        if e.code == 404:
+            logger.info('Object not found in cache.')
+        else:
+            logger.info('Problem to connect to shacache.')
+        return 1
+    except Exception:
+        logger.critical('Error while looking egg %s version %s', name, version,
+            exc_info=True)
+        return 1
+
+    return 0

slapos/cli/cache_source.py → slapos/cli/cache_url.py

@@ -33,9 +33,10 @@ import json
 import re
 import requests
 import sys
-
 import prettytable
 
+from six.moves.urllib.error import HTTPError
+
 from slapos.grid import networkcache
 from slapos.cli.config import ConfigCommand
 from slapos.cli.command import resetLogger
@@ -47,6 +48,7 @@ class CacheLookupCommand(ConfigCommand):
 
     Check if source URL is available to be downloaded from cache.
     """
+    command_group = 'cachelookup'
 
     def get_parser(self, prog_name):
         ap = super(CacheLookupCommand, self).get_parser(prog_name)
@@ -56,43 +58,45 @@ class CacheLookupCommand(ConfigCommand):
 
     def take_action(self, args):
         configp = self.fetch_config(args)
-        cache_dir = configp.get('networkcache', 'download-cache-url')
-        sys.exit(do_lookup(self.app.log, cache_dir, args.url))
+        cache_dir = configp.get('networkcache', 'download-dir-url')
+        cache_url = configp.get('networkcache', 'download-cache-url')
+        signature_certificate_list = configp.get('networkcache', 'signature-certificate-list')
 
-def do_lookup(logger, cache_dir, url):
-    md5 = hashlib.md5(str2bytes(url)).hexdigest()
+        sys.exit(
+          do_lookup(self.app.log, cache_dir, cache_url,
+                    signature_certificate_list, args.url))
 
+def do_lookup(logger, cache_dir, cache_url, signature_certificate_list,
+              url):
+    key = 'file-urlmd5:' + hashlib.md5(url.encode()).hexdigest()
     try:
-        cached_url = '%s/slapos-buildout-%s' % (cache_dir, md5)
-        logger.debug('Connecting to %s', url)
-        req = requests.get(cached_url, timeout=5)
-    except (requests.Timeout, requests.ConnectionError):
-        logger.critical('Cannot connect to cache server at %s', cached_url)
-        return 10
-
-    if not req.ok:
-        if req.status_code == 404:
-            logger.critical('Object not in cache: %s', url)
-        else:
-            logger.critical('Error while looking object %s: %s', url, req.reason)
-        return 10
-
-    entries = req.json()
-
+        entries = networkcache.download_entry_list(cache_url, cache_dir,
+            key, logger, signature_certificate_list)
         if not entries:
-        logger.info('Object found in cache, but has no entries.')
+            logger.info('Object found in cache, but has no entry.')
             return 0
 
-    pt = prettytable.PrettyTable(['file', 'sha512'])
-    entry_list = sorted(json.loads(entry[0]) for entry in entries)
-    for entry in entry_list:
-        pt.add_row([entry["file"], entry["sha512"]])
+        pt = prettytable.PrettyTable(['url', 'sha512', 'signed'])
+        for entry in entries:
+            d = json.loads(entry[0])
+            pt.add_row([d["url"], d["sha512"], entry[1]])
 
-    meta = json.loads(entries[0][0])
         logger.info('Software source URL: %s', url)
-    logger.info('SHADIR URL: %s', cached_url)
+        logger.info('SHADIR URL: %s/%s\n', cache_dir, key)
 
         resetLogger(logger)
         for line in pt.get_string(border=True, padding_width=0, vrules=prettytable.NONE).split('\n'):
             logger.info(line)
+    except HTTPError as e:
+        if e.code == 404:
+            logger.info('Object not found in cache.')
+        else:
+            logger.info('Problem to connect to shacache.')
+        return 1
+    except Exception:
+        logger.critical('Error while looking object %s', url,
+            exc_info=True)
+        return 1
+
+
     return 0

slapos/grid/networkcache.py

@@ -76,7 +76,7 @@ def loadJsonEntry(jentry):
 
 
 def download_entry_list(cache_url, dir_url, key, logger,
-                        signature_certificate_list, software_url):
+                        signature_certificate_list):
     nc = NetworkcacheClient(cache_url, dir_url,
         signature_certificate_list=signature_certificate_list or None)
     entry_list = nc.select_generic(key, filter=False)