client: Allow to force TLS via neos:// scheme

Similarly to how it is done with e.g. http:// and https:// - if neos://
is given TLS usage is forced and ca/cert/key must be there either in the
URI itself, or in $NEO_CA, $NEO_CERT and $NEO_KEY environment variables
mimicking the way how e.g. for https:// TLS credentials are taken from
host environment, not from the uri.

The latter might be usability convenience, but is also useful for WCFS
which needs to be able to remove secrets from uri on zurl normalization.

Please see discussion at nexedi/neoppod!18 (comment 184439)
for details.

/cc @levin.zimmermann
/reviewed-by @jm
/reviewed-on nexedi/neoppod!21

neo/client/zodburi.py

@@ -18,11 +18,13 @@
 URI format:
 
     neo://name@master1,master2,...,masterN?options
+    neos://----//---- with $NEO_CA, $NEO_CERT and $NEO_KEY providing defaults for ca/cert/key options
 """
 
 import ZODB.config
 import ZConfig
 
+import os
 from cStringIO import StringIO
 from collections import OrderedDict
 from urlparse import urlsplit, parse_qsl
@@ -52,8 +54,8 @@ def canonical_opt_name(name):
 def _resolve_uri(uri):
     scheme, netloc, path, query, frag = urlsplit(uri)
 
-    if scheme != "neo":
-        raise ValueError("invalid uri: %s : expected neo:// scheme" % uri)
+    if scheme not in ("neo", "neos"):
+        raise ValueError("invalid uri: %s : expected neo:// or neos:// scheme" % uri)
     if path != "":
         raise ValueError("invalid uri: %s : non-empty path" % uri)
     if frag != "":
@@ -90,6 +92,19 @@ def _resolve_uri(uri):
             else:
                 dbkw[k] = v
 
+    # neo://   use TLS only if ca/cert/key are explicitly specified in uri
+    # neos://  force TLS to be used and take ca/cert/key from environment if
+    #          TLS credentials are not explicitly specified in uri
+    if scheme == "neos":
+        for k in ('ca', 'cert', 'key'):
+            if k in neokw:
+                continue
+            kenv = "NEO_" + k.upper()
+            v = os.environ.get(kenv)
+            if not v:
+                raise ValueError("invalid uri: %s : option %s not specified "
+                                 "and $%s is also not set" % (uri, k, kenv))
+            setopt(k, v)
 
     # now we have everything. Let ZConfig do actual work for validation options
     # and borning the storage

neo/tests/client/testZODBURI.py

@@ -14,27 +14,28 @@
 # You should have received a copy of the GNU General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 
+import os
 import unittest
 from neo.client.zodburi import _resolve_uri
 
 
 testv = [
-    # [] of (uri, zconf_ok, dbkw_ok)
-    ("neo://dbname@master",
+    # [] of (uri, env, zconf_ok, dbkw_ok)
+    ("neo://dbname@master", {},
      """\
   master_nodes\tmaster
   name\tdbname
 """,
      {}),
 
-    ("neo://db2@master1:port1,master2:port2,master3:port3",
+    ("neo://db2@master1:port1,master2:port2,master3:port3", {},
      """\
   master_nodes\tmaster1:port1 master2:port2 master3:port3
   name\tdb2
 """,
      {}),
 
-    ("neo://db3@master1,master2:port2?read_only=true",
+    ("neo://db3@master1,master2:port2?read_only=true", {},
      """\
   master_nodes\tmaster1 master2:port2
   name\tdb3
@@ -44,7 +45,7 @@ testv = [
 
     ("neo://db4@[2001:67c:1254:2a::1]:1234,master2:port2?read_only=false"
      "&compress=true&logfile=xxx&alpha=111&dynamic_master_list=zzz&ca=qqq"
-     "&cert=rrr&key=sss&beta=222",
+     "&cert=rrr&key=sss&beta=222", {},
      """\
   master_nodes\t[2001:67c:1254:2a::1]:1234 master2:port2
   name\tdb4
@@ -57,6 +58,64 @@ testv = [
   key\tsss
 """,
      {"alpha": "111", "beta": "222"}),
+
+    ("neos://db5@master?ca=~/path/to/ca&cert=/path/to/cert&key=key.crt&logfile=xxx", {},
+     """\
+  master_nodes\tmaster
+  name\tdb5
+  ca\t~/path/to/ca
+  cert\t/path/to/cert
+  key\tkey.crt
+  logfile\txxx
+""",
+     {}),
+
+    ("neos://db6@master?ca=~/path/to/ca&cert=/path/to/cert&key=key.crt&logfile=xxx",
+     {"NEO_CA": "/ca.crt", "NEO_CERT": "/cert.crt", "NEO_KEY": "/key.crt"},
+     """\
+  master_nodes\tmaster
+  name\tdb6
+  ca\t~/path/to/ca
+  cert\t/path/to/cert
+  key\tkey.crt
+  logfile\txxx
+""",
+     {}),
+
+    ("neos://db7@master?ca=~/path/to/ca&key=key.crt&logfile=xxx",
+     {"NEO_CA": "/ca.crt", "NEO_CERT": "/cert.crt", "NEO_KEY": "/key.crt"},
+     """\
+  master_nodes\tmaster
+  name\tdb7
+  ca\t~/path/to/ca
+  key\tkey.crt
+  logfile\txxx
+  cert\t/cert.crt
+""",
+     {}),
+
+    ("neos://db8@master?logfile=xxx",
+     {"NEO_CA": "/ca.crt", "NEO_CERT": "/cert.crt", "NEO_KEY": "/key.crt"},
+     """\
+  master_nodes\tmaster
+  name\tdb8
+  logfile\txxx
+  ca\t/ca.crt
+  cert\t/cert.crt
+  key\t/key.crt
+""",
+     {}),
+
+    ("neos://db9@master",
+     {"NEO_CA": "/ca.crt", "NEO_CERT": "/cert.crt", "NEO_KEY": "/key.crt"},
+     """\
+  master_nodes\tmaster
+  name\tdb9
+  ca\t/ca.crt
+  cert\t/cert.crt
+  key\t/key.crt
+""",
+     {}),
 ]
 
 
@@ -77,11 +136,16 @@ class ZODBURITests(unittest.TestCase):
         self.assertRaises(ValueError, _resolve_uri, "neo://db@master?name=zzz")
 
         # verify zodburi resolver produces expected zconfig
-        for uri, zconf_ok, dbkw_ok in testv:
+        for uri, env, zconf_ok, dbkw_ok in testv:
             zconf_ok = "%import neo.client\n<NEOStorage>\n" + zconf_ok + \
                        "</NEOStorage>\n"
 
+            envsave = os.environ
+            os.environ = env
+            try:
                 zconf, dbkw = _resolve_uri(uri)
+            finally:
+                os.environ = envsave
             self.assertMultiLineEqual(zconf, zconf_ok)
             self.assertEqual(dbkw, dbkw_ok)
 

setup.py

@@ -108,6 +108,7 @@ setup(
         ],
         'zodburi.resolvers': [
             'neo = neo.client.zodburi:resolve_uri [client]',
+            'neos = neo.client.zodburi:resolve_uri [client]',
         ],
     },
     install_requires = [