Skip to content

N
neoppod
Commit 95572d6a authored by Kirill Smelkov's avatar Kirill Smelkov
Browse files
Options

X: Sync zurl format with NEO/py 

Hello Kirill,

in nexedi/neoppod!18 and nexedi/neoppod!21 we could find a common solution for a zurl format that previously diverged between NEOgo and NEOpy. The purpose of this MR is to sync again NEOgo and NEOpy zurl format. After merging this, we can continue to sync NEO zurl format in 'wendelin.core' & 'slapos'. Then we finally have unified approach again, which simplifies understanding and reduces unnecessary mental overhead.

As this is strongly related to nexedi/neoppod!21 I thought it'd be a good idea to generally reduce difference and to replace WIP commits with merged NEOpy upstream commits.

Best, Levin

/reviewed-by @kirr
/reviewed-on !7

* lev/sync-zurl:
  client: Don't allow oPtion_nAme in zurl
  app: Remember SSL credentials so that it is possible to retrieve them
  client: Allow to force TLS via neos:// scheme
  client: Don't allow master_nodes and name to be present in options
  Revert "."
  Revert "Y client: Fix URI scheme to move credentials out of query"
  Revert "X Adjust NEO/go to neo:// URL change + py fixups"
  Revert "fixup! Y client: Fix URI scheme to move credentials out of query"
  Revert "Y client: Don't allow master_nodes and name to be present in options"
  go/client/zurl: Sync format to py upstream
parents 1ad088c8 3877e259
Hide whitespace changes
Inline Side-by-side
Showing with 184 additions and 138 deletions
go/neo/client.go
View file @ 95572d6a
// Copyright (C) 2017-2023 Nexedi SA and Contributors. // Copyright (C) 2017-2024 Nexedi SA and Contributors.
// Kirill Smelkov <kirr@nexedi.com> // Kirill Smelkov <kirr@nexedi.com>
// //
// This program is free software: you can Use, Study, Modify and Redistribute // This program is free software: you can Use, Study, Modify and Redistribute
...@@ -484,7 +484,7 @@ func openClientByURL(ctx context.Context, u *url.URL, opt *zodb.DriverOptions) ( ...@@ -484,7 +484,7 @@ func openClientByURL(ctx context.Context, u *url.URL, opt *zodb.DriverOptions) (
// parseURL extracts information from a NEO URI and puts this information into // parseURL extracts information from a NEO URI and puts this information into
// a urlInfo. // a urlInfo.
func parseURL(ctx context.Context, u *url.URL) (urlinfo *urlInfo, err error) { func parseURL(ctx context.Context, u *url.URL) (urlinfo *urlInfo, err error) {
// neo(s)://[credentials@]master1,master2,...,masterN/name?options // neo(s)://name@master1,master2,...,masterN?options
var ssl bool var ssl bool
switch u.Scheme { switch u.Scheme {
...@@ -493,50 +493,35 @@ func parseURL(ctx context.Context, u *url.URL) (urlinfo *urlInfo, err error) { ...@@ -493,50 +493,35 @@ func parseURL(ctx context.Context, u *url.URL) (urlinfo *urlInfo, err error) {
default: return nil, fmt.Errorf("invalid scheme") default: return nil, fmt.Errorf("invalid scheme")
} }
cred := u.User.String() name := u.User.String()
// ca=ca.crt;cert=my.crt;key=my.key if name == "" {
cred = strings.ReplaceAll(cred, ";", "&") // ; is no longer in default separators set https://github.com/golang/go/issues/25192 return nil, fmt.Errorf("cluster name not specified")
x, err := xurl.ParseQuery(cred) }
q, err := xurl.ParseQuery(u.RawQuery)
if err != nil { if err != nil {
return nil, fmt.Errorf("credentials: %s", err) return nil, err
} }
// xpop pops k from credentials, defaulting to $NEO_<K> if envok.
xpop := func(k string, envok bool) string { // qpop pops k from query, defaulting to $NEO_<K> if envok.
v, ok := x[k] qpop := func(k string, envok bool) string {
v, ok := q[k]
if !ok && envok { if !ok && envok {
v = os.Getenv("NEO_"+strings.ToUpper(k)) v = os.Getenv("NEO_"+strings.ToUpper(k))
} }
delete(x, k) delete(q, k)
return v return v
} }
netcfg := neonet.Config{} netcfg := neonet.Config{}
netcfg.LoNode = xpop("lonode", false) netcfg.LoNode = qpop("lonode", false)
if !ssl { // neos:// force TLS to be used and take ca/cert/key from environment if
if len(x) != 0 { // TLS credentials are not explicitly specified in uri
return nil, fmt.Errorf("credentials can be specified only with neos:// scheme") // neo:// use TLS only if ca/cert/key are explicitly specified in uri
} netcfg.CA = qpop("ca", ssl)
} else { netcfg.Cert = qpop("cert", ssl)
netcfg.CA = xpop("ca", true) netcfg.Key = qpop("key", ssl)
netcfg.Cert = xpop("cert", true)
netcfg.Key = xpop("key", true)
if len(x) != 0 {
return nil, fmt.Errorf("invalid credentials: %v", x)
}
}
name := u.Path
name = strings.TrimPrefix(name, "/")
if name == "" {
return nil, fmt.Errorf("cluster name not specified")
}
q, err := xurl.ParseQuery(u.RawQuery)
if err != nil {
return nil, err
}
// pop not yet used client options // pop not yet used client options
// (our neo client doesn't apply their effect yet) // (our neo client doesn't apply their effect yet)
......
go/neo/client_test.go
View file @ 95572d6a
// Copyright (C) 2020-2023 Nexedi SA and Contributors. // Copyright (C) 2020-2024 Nexedi SA and Contributors.
// Kirill Smelkov <kirr@nexedi.com> // Kirill Smelkov <kirr@nexedi.com>
// //
// This program is free software: you can Use, Study, Modify and Redistribute // This program is free software: you can Use, Study, Modify and Redistribute
...@@ -175,7 +175,7 @@ func (n *NEOPySrv) clusterName() string { ...@@ -175,7 +175,7 @@ func (n *NEOPySrv) clusterName() string {
} }
func (n *NEOPySrv) URL() string { func (n *NEOPySrv) URL() string {
return fmt.Sprintf("%s%s/%s", n.opt.URLPrefix(), strings.Join(n.masterAddrSlice, ","), n.clusterName()) return fmt.Sprintf("%s%s%s", n.opt.URLPrefix(), strings.Join(n.masterAddrSlice, ","), n.opt.URLQuery())
} }
func (n *NEOPySrv) LogTail() (string, error) { func (n *NEOPySrv) LogTail() (string, error) {
...@@ -361,7 +361,7 @@ func (n *NEOGoSrv) masterAddrSlice() []string { ...@@ -361,7 +361,7 @@ func (n *NEOGoSrv) masterAddrSlice() []string {
} }
func (n *NEOGoSrv) URL() string { func (n *NEOGoSrv) URL() string {
return fmt.Sprintf("%s%s/%s", n.opt.URLPrefix(), strings.Join(n.masterAddrSlice(), ","), n.opt.name) return fmt.Sprintf("%s%s%s", n.opt.URLPrefix(), strings.Join(n.masterAddrSlice(), ","), n.opt.URLQuery())
} }
...@@ -391,22 +391,35 @@ func (opt NEOSrvOptions) Key() string { ...@@ -391,22 +391,35 @@ func (opt NEOSrvOptions) Key() string {
} }
// URLPrefix returns start of URL for a NEO server started with opt. // URLPrefix returns start of URL for a NEO server started with opt.
// e.g. neo:// or neos://ca=1;cert=2;key=3@ // e.g. neo://test@ or neos://test@
// To be come complete returned URL has to be appended with host and path parts. // To become complete returned URL has to be appended with host and query parts.
func (opt NEOSrvOptions) URLPrefix() string { func (opt NEOSrvOptions) URLPrefix() string {
zurl := "" zurl := ""
if !opt.SSL { if !opt.SSL {
zurl = "neo://" zurl = "neo://"
} else { } else {
zurl = "neos://" zurl = "neos://"
zurl += "ca=" + url.QueryEscape(opt.CA()) +";"
zurl += "cert=" + url.QueryEscape(opt.Cert()) +";"
zurl += "key=" + url.QueryEscape(opt.Key())
zurl += "@"
} }
zurl += opt.name
zurl += "@"
return zurl return zurl
} }
// URLQuery returns query part of the URL for a NEO server started with opt.
// e.g. ?ca=...&cert=...&key=...
func (opt NEOSrvOptions) URLQuery() string {
query := ""
if opt.SSL {
query += "ca=" + url.QueryEscape(opt.CA()) +"&"
query += "cert=" + url.QueryEscape(opt.Cert()) +"&"
query += "key=" + url.QueryEscape(opt.Key())
}
if query != "" {
query = "?" + query
}
return query
}
// ---------------- // ----------------
...@@ -677,14 +690,14 @@ func TestWatch(t *testing.T) { ...@@ -677,14 +690,14 @@ func TestWatch(t *testing.T) {
// scheme neo(s)://[credentials@]master1,master2,...,masterN/name?options) // scheme neo(s)://[credentials@]master1,master2,...,masterN/name?options)
func TestParseURL(t *testing.T) { func TestParseURL(t *testing.T) {
// Most simple valid URI // Most simple valid URI
testParseURL(t, "neo://127.0.0.1/test", urlInfo{}) testParseURL(t, "neo://test@127.0.0.1", urlInfo{})
// With 2 masters // With 2 masters
testParseURL(t, "neo://127.0.0.1,127.0.0.2/test", urlInfo{masterAddr: "127.0.0.1,127.0.0.2"}) testParseURL(t, "neo://test@127.0.0.1,127.0.0.2", urlInfo{masterAddr: "127.0.0.1,127.0.0.2"})
// With ssl // With ssl
u := "neos://ca=ca;cert=cert;key=key@127.0.0.1/test" u := "neos://test@127.0.0.1?ca=ca&cert=cert&key=key"
testParseURL(t, u, urlInfo{netcfg: neonet.Config{CA: "ca", Cert: "cert", Key: "key"}}) testParseURL(t, u, urlInfo{netcfg: neonet.Config{CA: "ca", Cert: "cert", Key: "key"}})
// With query parameters // With query parameters
u = "neo://127.0.0.1/test?compress=true&logfile=n.log&cache-size=256" u = "neo://test@127.0.0.1/compress=true&logfile=n.log&cache-size=256"
testParseURL(t, u, urlInfo{}) testParseURL(t, u, urlInfo{})
} }
......
go/neo/t/neotest
View file @ 95572d6a
...@@ -499,7 +499,7 @@ GENsqlite() { ...@@ -499,7 +499,7 @@ GENsqlite() {
NEOpylite NEOpylite
# NOTE compression is disabled because when benchmarking server latency # NOTE compression is disabled because when benchmarking server latency
# we do not want the time client(s) take to decompress data to interfere. # we do not want the time client(s) take to decompress data to interfere.
${dataset}_gen_data neo://$Mbind/$neocluster?compress=false $dataset_size ${dataset}_gen_data neo://$neocluster@$Mbind?compress=false $dataset_size
xneoctl set cluster stopping xneoctl set cluster stopping
wait # XXX fragile - won't work if there are children spawned outside wait # XXX fragile - won't work if there are children spawned outside
sync sync
...@@ -512,7 +512,7 @@ GENsql() { ...@@ -512,7 +512,7 @@ GENsql() {
echo -e '\n*** generating sql data...' echo -e '\n*** generating sql data...'
NEOpysql NEOpysql
# NOTE compression is disabled - see ^^^ (sqlite) for rationale. # NOTE compression is disabled - see ^^^ (sqlite) for rationale.
${dataset}_gen_data neo://$Mbind/$neocluster?compress=false $dataset_size ${dataset}_gen_data neo://$neocluster@$Mbind?compress=false $dataset_size
xneoctl set cluster stopping xneoctl set cluster stopping
sleep 1 # XXX fragile sleep 1 # XXX fragile
xmysql -e "SHUTDOWN" xmysql -e "SHUTDOWN"
...@@ -1194,21 +1194,21 @@ zbench_local() { ...@@ -1194,21 +1194,21 @@ zbench_local() {
# XXX save time - we show only neo/py(!log)/sqlite # XXX save time - we show only neo/py(!log)/sqlite
#echo -e "\n*** NEO/py sqlite" #echo -e "\n*** NEO/py sqlite"
#NEOpylite #NEOpylite
#zbench neo://$Mbind/$neocluster neo/py/sqlite·P$Pneo $zhashok #zbench neo://$neocluster@$Mbind neo/py/sqlite·P$Pneo $zhashok
#xneoctl set cluster stopping #xneoctl set cluster stopping
#wait #wait
# XXX JM asked to also have NEO/py with logging disabled # XXX JM asked to also have NEO/py with logging disabled
echo -e "\n*** NEO/py sqlite (logging disabled)" echo -e "\n*** NEO/py sqlite (logging disabled)"
X_NEOPY_LOG_SKIP=y NEOpylite X_NEOPY_LOG_SKIP=y NEOpylite
zbench neo://$Mbind/$neocluster "neo/py(!log)/sqlite·P$Pneo" $zhashok zbench neo://$neocluster@$Mbind "neo/py(!log)/sqlite·P$Pneo" $zhashok
xneoctl set cluster stopping xneoctl set cluster stopping
wait wait
# XXX save time - we show only neo/py(!log)/sql # XXX save time - we show only neo/py(!log)/sql
#echo -e "\n*** NEO/py sql" #echo -e "\n*** NEO/py sql"
#NEOpysql #NEOpysql
#zbench neo://$Mbind/$neocluster neo/py/sql·P$Pneo $zhashok #zbench neo://$neocluster@$Mbind neo/py/sql·P$Pneo $zhashok
#xneoctl set cluster stopping #xneoctl set cluster stopping
#xmysql -e "SHUTDOWN" #xmysql -e "SHUTDOWN"
#wait #wait
...@@ -1216,27 +1216,27 @@ zbench_local() { ...@@ -1216,27 +1216,27 @@ zbench_local() {
# XXX JM asked to also have NEO/py with logging disabled # XXX JM asked to also have NEO/py with logging disabled
echo -e "\n*** NEO/py sql (logging disabled)" echo -e "\n*** NEO/py sql (logging disabled)"
X_NEOPY_LOG_SKIP=y NEOpysql X_NEOPY_LOG_SKIP=y NEOpysql
zbench neo://$Mbind/$neocluster "neo/py(!log)/sql·P$Pneo" $zhashok zbench neo://$neocluster@$Mbind "neo/py(!log)/sql·P$Pneo" $zhashok
xneoctl set cluster stopping xneoctl set cluster stopping
xmysql -e "SHUTDOWN" xmysql -e "SHUTDOWN"
wait wait
echo -e "\n*** NEO/go fs1" echo -e "\n*** NEO/go fs1"
NEOgofs1 NEOgofs1
zbench neo://$Mbind/$neocluster neo/go/fs1·P1 $zhashok zbench neo://$neocluster@$Mbind neo/go/fs1·P1 $zhashok
xneoctl set cluster stopping xneoctl set cluster stopping
wait wait
echo -e "\n*** NEO/go fs1 (sha1 disabled on: storage, client)" echo -e "\n*** NEO/go fs1 (sha1 disabled on: storage, client)"
X_NEOGO_SHA1_SKIP=y NEOgofs1 X_NEOGO_SHA1_SKIP=y NEOgofs1
X_NEOGO_SHA1_SKIP=y zbench_go neo://$Mbind/$neocluster "neo/go/fs1(!sha1)·P1" $zhashok X_NEOGO_SHA1_SKIP=y zbench_go neo://$neocluster@$Mbind "neo/go/fs1(!sha1)·P1" $zhashok
xneoctl set cluster stopping xneoctl set cluster stopping
wait wait
echo -e "\n*** NEO/go sqlite" echo -e "\n*** NEO/go sqlite"
if [ $Pneo == 1 ]; then if [ $Pneo == 1 ]; then
NEOgolite NEOgolite
zbench neo://$Mbind/$neocluster@ neo/go/sqlite·P$Pneo $zhashok zbench neo://$neocluster@$Mbind neo/go/sqlite·P$Pneo $zhashok
xneoctl set cluster stopping xneoctl set cluster stopping
wait wait
else else
...@@ -1246,7 +1246,7 @@ zbench_local() { ...@@ -1246,7 +1246,7 @@ zbench_local() {
echo -e "\n*** NEO/go sqlite (sha1 disabled on: client)" echo -e "\n*** NEO/go sqlite (sha1 disabled on: client)"
if [ $Pneo == 1 ]; then if [ $Pneo == 1 ]; then
NEOgolite NEOgolite
X_NEOGO_SHA1_SKIP=y zbench_go neo://$Mbind/$neocluster "neo/go/sqlite·P$Pneo" $zhashok X_NEOGO_SHA1_SKIP=y zbench_go neo://$neocluster@$Mbind "neo/go/sqlite·P$Pneo" $zhashok
xneoctl set cluster stopping xneoctl set cluster stopping
wait wait
else else
...@@ -1315,21 +1315,21 @@ zbench_cluster() { ...@@ -1315,21 +1315,21 @@ zbench_cluster() {
# XXX save time - we show only neo/py(!log)/sqlite # XXX save time - we show only neo/py(!log)/sqlite
#echo -e "\n*** NEO/py sqlite" #echo -e "\n*** NEO/py sqlite"
#NEOpylite #NEOpylite
#on $url ./neotest zbench-client neo://$Mbind/$neocluster neo/py/sqlite·P$Pneo $zhashok #on $url ./neotest zbench-client neo://$neocluster@$Mbind neo/py/sqlite·P$Pneo $zhashok
#xneoctl set cluster stopping #xneoctl set cluster stopping
#wait #wait
# XXX JM asked to also have NEO/py with logging disabled # XXX JM asked to also have NEO/py with logging disabled
echo -e "\n*** NEO/py sqlite (logging disabled)" echo -e "\n*** NEO/py sqlite (logging disabled)"
X_NEOPY_LOG_SKIP=y NEOpylite X_NEOPY_LOG_SKIP=y NEOpylite
on $url ./neotest zbench-client neo://$Mbind/$neocluster "\\\"neo/py(!log)/sqlite\\\"·P$Pneo" $zhashok on $url ./neotest zbench-client neo://$neocluster@$Mbind "\\\"neo/py(!log)/sqlite\\\"·P$Pneo" $zhashok
xneoctl set cluster stopping xneoctl set cluster stopping
wait wait
# XXX save time - we show only neo/py(!log)/sql # XXX save time - we show only neo/py(!log)/sql
#echo -e "\n*** NEO/py sql" #echo -e "\n*** NEO/py sql"
#NEOpysql #NEOpysql
#on $url ./neotest zbench-client neo://$Mbind/$neocluster neo/py/sql·P$Pneo $zhashok #on $url ./neotest zbench-client neo://$neocluster@$Mbind neo/py/sql·P$Pneo $zhashok
#xneoctl set cluster stopping #xneoctl set cluster stopping
#xmysql -e "SHUTDOWN" #xmysql -e "SHUTDOWN"
#wait #wait
...@@ -1337,27 +1337,27 @@ zbench_cluster() { ...@@ -1337,27 +1337,27 @@ zbench_cluster() {
# XXX JM asked to also have NEO/py with logging disabled # XXX JM asked to also have NEO/py with logging disabled
echo -e "\n*** NEO/py sql (logging disabled)" echo -e "\n*** NEO/py sql (logging disabled)"
X_NEOPY_LOG_SKIP=y NEOpysql X_NEOPY_LOG_SKIP=y NEOpysql
on $url ./neotest zbench-client neo://$Mbind/$neocluster "\\\"neo/py(!log)/sql\\\"·P$Pneo" $zhashok on $url ./neotest zbench-client neo://$neocluster@$Mbind "\\\"neo/py(!log)/sql\\\"·P$Pneo" $zhashok
xneoctl set cluster stopping xneoctl set cluster stopping
xmysql -e "SHUTDOWN" xmysql -e "SHUTDOWN"
wait wait
echo -e "\n*** NEO/go fs" echo -e "\n*** NEO/go fs"
NEOgofs1 NEOgofs1
on $url ./neotest zbench-client neo://$Mbind/$neocluster neo/go/fs1·P1 $zhashok on $url ./neotest zbench-client neo://$neocluster@$Mbind neo/go/fs1·P1 $zhashok
xneoctl set cluster stopping xneoctl set cluster stopping
wait wait
echo -e "\n*** NEO/go fs1 (sha1 disabled on: storage, client)" echo -e "\n*** NEO/go fs1 (sha1 disabled on: storage, client)"
X_NEOGO_SHA1_SKIP=y NEOgofs1 X_NEOGO_SHA1_SKIP=y NEOgofs1
on $url X_NEOGO_SHA1_SKIP=y ./neotest zbench-client --goonly neo://$Mbind/$neocluster "\\\"neo/go/fs1(!sha1)\\\"·P1" $zhashok on $url X_NEOGO_SHA1_SKIP=y ./neotest zbench-client --goonly neo://$neocluster@$Mbind "\\\"neo/go/fs1(!sha1)\\\"·P1" $zhashok
xneoctl set cluster stopping xneoctl set cluster stopping
wait wait
echo -e "\n*** NEO/go sqlite" echo -e "\n*** NEO/go sqlite"
if [ $Pneo == 1 ]; then if [ $Pneo == 1 ]; then
NEOgolite NEOgolite
on $url ./neotest zbench-client neo://$Mbind/$neocluster neo/go/sqlite·P$Pneo $zhashok on $url ./neotest zbench-client neo://$neocluster@$Mbind neo/go/sqlite·P$Pneo $zhashok
xneoctl set cluster stopping xneoctl set cluster stopping
wait wait
else else
...@@ -1367,7 +1367,7 @@ zbench_cluster() { ...@@ -1367,7 +1367,7 @@ zbench_cluster() {
echo -e "\n*** NEO/go sqlite (sha1 disabled on: client)" echo -e "\n*** NEO/go sqlite (sha1 disabled on: client)"
if [ $Pneo == 1 ]; then if [ $Pneo == 1 ]; then
NEOgolite NEOgolite
on $url X_NEOGO_SHA1_SKIP=y ./neotest zbench-client --goonly neo://$Mbind/$neocluster "\\\"neo/go/sqlite\\\"·P$Pneo" $zhashok on $url X_NEOGO_SHA1_SKIP=y ./neotest zbench-client --goonly neo://$neocluster@$Mbind "\\\"neo/go/sqlite\\\"·P$Pneo" $zhashok
xneoctl set cluster stopping xneoctl set cluster stopping
wait wait
else else
......
neo/client/zodburi.py
View file @ 95572d6a
# #
# Copyright (C) 2017-2020 Nexedi SA # Copyright (C) 2017-2019 Nexedi SA
# #
# This program is free software; you can redistribute it and/or # This program is free software; you can redistribute it and/or
# modify it under the terms of the GNU General Public License # modify it under the terms of the GNU General Public License
...@@ -17,19 +17,18 @@ ...@@ -17,19 +17,18 @@
URI format: URI format:
neo(s)://[credentials@]master1,master2,...,masterN/name?options neo://name@master1,master2,...,masterN?options
neos://----//---- with $NEO_CA, $NEO_CERT and $NEO_KEY providing defaults for ca/cert/key options
""" """
import ZODB.config import ZODB.config
import ZConfig import ZConfig
import os
from cStringIO import StringIO from cStringIO import StringIO
from collections import OrderedDict from collections import OrderedDict
from urlparse import urlsplit, parse_qsl from urlparse import urlsplit, parse_qsl
# _credopts defines which options correspond to credentials
_credopts = {'ca', 'cert', 'key'}
# neo_zconf_options returns set of zconfig options supported by NEO storage # neo_zconf_options returns set of zconfig options supported by NEO storage
def neo_zconf_options(): def neo_zconf_options():
neo_schema = """<schema> neo_schema = """<schema>
...@@ -44,75 +43,64 @@ def neo_zconf_options(): ...@@ -44,75 +43,64 @@ def neo_zconf_options():
options = {k for k, _ in neo_storage_zconf} options = {k for k, _ in neo_storage_zconf}
assert 'master_nodes' in options assert 'master_nodes' in options
assert 'name' in options assert 'name' in options
options.remove('master_nodes') # comes in netloc
options.remove('name') # comes in path
for opt in _credopts:
assert opt in options, opt
return options return options
# canonical_opt_name returns "oPtion_nAme" as "option-name"
def canonical_opt_name(name):
return name.lower().replace('_', '-')
# worker entrypoint for resolve_uri and tests # worker entrypoint for resolve_uri and tests
def _resolve_uri(uri): def _resolve_uri(uri):
scheme, netloc, path, query, frag = urlsplit(uri) scheme, netloc, path, query, frag = urlsplit(uri)
if scheme not in ("neo", "neos"): if scheme not in ("neo", "neos"):
raise ValueError("invalid uri: %s : expected neo:// or neos:// scheme" % uri) raise ValueError("invalid uri: %s : expected neo:// or neos:// scheme" % uri)
if path != "":
raise ValueError("invalid uri: %s : non-empty path" % uri)
if frag != "": if frag != "":
raise ValueError("invalid uri: %s : non-empty fragment" % uri) raise ValueError("invalid uri: %s : non-empty fragment" % uri)
# name is given as path # extract master list and name from netloc
if path.startswith("/"): name, masterloc = netloc.split('@', 1)
path = path[1:]
name = path
if name == '':
raise ValueError("invalid uri: %s : cluster name not specified" % uri)
# extract master list and credentials from netloc
cred, masterloc = '', netloc
if '@' in netloc:
cred, masterloc = netloc.split('@', 1)
master_list = masterloc.split(',') master_list = masterloc.split(',')
neokw = OrderedDict() neokw = OrderedDict()
neokw['master_nodes'] = ' '.join(master_list) neokw['master_nodes'] = ' '.join(master_list)
neokw['name'] = name neokw['name'] = name
def setopt(k, v):
# parse credentials if k in ('master_nodes', 'name'):
if cred: raise ValueError("invalid uri: %s : invalid option %s" % (uri, k))
if scheme != "neos": neokw[k] = v
raise ValueError("invalid uri: %s : credentials can be specified only with neos:// scheme" % uri)
# ca=ca.crt;cert=my.crt;key=my.key
cred = cred.replace(';', '&') # ; is no longer in default separators set bugs.python.org/issue42967
for k, v in OrderedDict(parse_qsl(cred)).items():
if k not in _credopts:
raise ValueError("invalid uri: %s : unexpected credential %s" % (uri, k))
neokw[k] = v
# get options from query: only those that are defined by NEO schema go to # get options from query: only those that are defined by NEO schema go to
# storage - rest are returned as database options # storage - rest are returned as database options
dbkw = {} dbkw = {}
neo_options = neo_zconf_options() neo_options = neo_zconf_options()
for k, v in OrderedDict(parse_qsl(query)).items(): for k, v in OrderedDict(parse_qsl(query)).items():
if k in _credopts: if k in neo_options:
raise ValueError("invalid uri: %s : option %s must be in credentials" % (uri, k)) setopt(k, v)
elif k in neo_options:
neokw[k] = v
else: else:
# it might be option for storage, but not in canonical form e.g. # it might be option for storage, but not in canonical form e.g.
# read_only -> read-only (zodburi world settled on using "_" and # read_only -> read-only (zodburi world settled on using "_" and
# ZConfig world on "-" as separators) # ZConfig world on "-" as separators)
k2 = canonical_opt_name(k) k2 = k.replace('_', '-')
if k2 in neo_options: if k2 in neo_options:
neokw[k2] = v setopt(k2, v)
# else keep this kv as db option # else keep this kv as db option
else: else:
dbkw[k] = v dbkw[k] = v
# neo:// use TLS only if ca/cert/key are explicitly specified in uri
# neos:// force TLS to be used and take ca/cert/key from environment if
# TLS credentials are not explicitly specified in uri
if scheme == "neos":
for k in ('ca', 'cert', 'key'):
if k in neokw:
continue
kenv = "NEO_" + k.upper()
v = os.environ.get(kenv)
if not v:
raise ValueError("invalid uri: %s : option %s not specified "
"and $%s is also not set" % (uri, k, kenv))
setopt(k, v)
# now we have everything. Let ZConfig do actual work for validation options # now we have everything. Let ZConfig do actual work for validation options
# and borning the storage # and borning the storage
......
neo/lib/app.py
View file @ 95572d6a
...@@ -72,7 +72,9 @@ class BaseApplication(object): ...@@ -72,7 +72,9 @@ class BaseApplication(object):
" provided: the CA certificate, and the certificate" " provided: the CA certificate, and the certificate"
" of this node with its private key.") " of this node with its private key.")
ca, cert, key = ssl ca, cert, key = ssl
self.ssl_credentials = ssl # remember ca/cert/key, so that zstor_2zurl could retrive them # remember ca/cert/key, so that zstor_2zurl in wendelin.core could
# retrieve them and fully reconstruct neos:// url of the storage
self.ssl_credentials = ssl
import ssl import ssl
version, version_name = max((getattr(ssl, k), k) version, version_name = max((getattr(ssl, k), k)
for k in dir(ssl) if k.startswith("PROTOCOL_TLSv")) for k in dir(ssl) if k.startswith("PROTOCOL_TLSv"))
......
neo/tests/client/testZODBURI.py
View file @ 95572d6a
# #
# Copyright (C) 2017-2020 Nexedi SA # Copyright (C) 2017-2019 Nexedi SA
# #
# This program is free software; you can redistribute it and/or # This program is free software; you can redistribute it and/or
# modify it under the terms of the GNU General Public License # modify it under the terms of the GNU General Public License
...@@ -14,27 +14,28 @@ ...@@ -14,27 +14,28 @@
# You should have received a copy of the GNU General Public License # You should have received a copy of the GNU General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>. # along with this program. If not, see <http://www.gnu.org/licenses/>.
import os
import unittest import unittest
from neo.client.zodburi import _resolve_uri from neo.client.zodburi import _resolve_uri
testv = [ testv = [
# [] of (uri, zconf_ok, dbkw_ok) # [] of (uri, env, zconf_ok, dbkw_ok)
("neo://master/dbname", ("neo://dbname@master", {},
"""\ """\
master_nodes\tmaster master_nodes\tmaster
name\tdbname name\tdbname
""", """,
{}), {}),
("neo://master1:port1,master2:port2,master3:port3/db2", ("neo://db2@master1:port1,master2:port2,master3:port3", {},
"""\ """\
master_nodes\tmaster1:port1 master2:port2 master3:port3 master_nodes\tmaster1:port1 master2:port2 master3:port3
name\tdb2 name\tdb2
""", """,
{}), {}),
("neo://master1,master2:port2/db3?read_only=true", ("neo://db3@master1,master2:port2?read_only=true", {},
"""\ """\
master_nodes\tmaster1 master2:port2 master_nodes\tmaster1 master2:port2
name\tdb3 name\tdb3
...@@ -42,21 +43,79 @@ testv = [ ...@@ -42,21 +43,79 @@ testv = [
""", """,
{}), {}),
("neos://ca=qqq;cert=rrr;key=sss@[2001:67c:1254:2a::1]:1234,master2:port2/db4?read_only=false" ("neo://db4@[2001:67c:1254:2a::1]:1234,master2:port2?read_only=false"
"&compress=true&logfile=xxx&alpha=111&dynamic_master_list=zzz" "&compress=true&logfile=xxx&alpha=111&dynamic_master_list=zzz&ca=qqq"
"&beta=222", "&cert=rrr&key=sss&beta=222", {},
"""\ """\
master_nodes\t[2001:67c:1254:2a::1]:1234 master2:port2 master_nodes\t[2001:67c:1254:2a::1]:1234 master2:port2
name\tdb4 name\tdb4
ca\tqqq
cert\trrr
key\tsss
read-only\tfalse read-only\tfalse
compress\ttrue compress\ttrue
logfile\txxx logfile\txxx
dynamic_master_list\tzzz dynamic_master_list\tzzz
ca\tqqq
cert\trrr
key\tsss
""", """,
{"alpha": "111", "beta": "222"}), {"alpha": "111", "beta": "222"}),
("neos://db5@master?ca=~/path/to/ca&cert=/path/to/cert&key=key.crt&logfile=xxx", {},
"""\
master_nodes\tmaster
name\tdb5
ca\t~/path/to/ca
cert\t/path/to/cert
key\tkey.crt
logfile\txxx
""",
{}),
("neos://db6@master?ca=~/path/to/ca&cert=/path/to/cert&key=key.crt&logfile=xxx",
{"NEO_CA": "/ca.crt", "NEO_CERT": "/cert.crt", "NEO_KEY": "/key.crt"},
"""\
master_nodes\tmaster
name\tdb6
ca\t~/path/to/ca
cert\t/path/to/cert
key\tkey.crt
logfile\txxx
""",
{}),
("neos://db7@master?ca=~/path/to/ca&key=key.crt&logfile=xxx",
{"NEO_CA": "/ca.crt", "NEO_CERT": "/cert.crt", "NEO_KEY": "/key.crt"},
"""\
master_nodes\tmaster
name\tdb7
ca\t~/path/to/ca
key\tkey.crt
logfile\txxx
cert\t/cert.crt
""",
{}),
("neos://db8@master?logfile=xxx",
{"NEO_CA": "/ca.crt", "NEO_CERT": "/cert.crt", "NEO_KEY": "/key.crt"},
"""\
master_nodes\tmaster
name\tdb8
logfile\txxx
ca\t/ca.crt
cert\t/cert.crt
key\t/key.crt
""",
{}),
("neos://db9@master",
{"NEO_CA": "/ca.crt", "NEO_CERT": "/cert.crt", "NEO_KEY": "/key.crt"},
"""\
master_nodes\tmaster
name\tdb9
ca\t/ca.crt
cert\t/cert.crt
key\t/key.crt
""",
{}),
] ]
...@@ -64,30 +123,29 @@ testv = [ ...@@ -64,30 +123,29 @@ testv = [
class ZODBURITests(unittest.TestCase): class ZODBURITests(unittest.TestCase):
def test_zodburi(self): def test_zodburi(self):
# invalid schema / fragment # invalid schema / path / fragment
self.assertRaises(ValueError, _resolve_uri, "http://master/db") self.assertRaises(ValueError, _resolve_uri, "http://db@master")
self.assertRaises(ValueError, _resolve_uri, "neo://master/db#frag") self.assertRaises(ValueError, _resolve_uri, "neo://db@master/path")
self.assertRaises(ValueError, _resolve_uri, "neo://db@master#frag")
# master/db not fully specified # db @ master not fully specified
self.assertRaises(ValueError, _resolve_uri, "neo://master") self.assertRaises(ValueError, _resolve_uri, "neo://master")
# master_nodes and name provided in option (they come in netloc and path) # master_nodes and name provided in options (they come in netloc)
self.assertRaises(ValueError, _resolve_uri, "neo://master/db?master_nodes=a,b,c") self.assertRaises(ValueError, _resolve_uri, "neo://db@master?master_nodes=a,b,c")
self.assertRaises(ValueError, _resolve_uri, "neo://master/db?name=zzz") self.assertRaises(ValueError, _resolve_uri, "neo://db@master?name=zzz")
# option that corresponds to credential provided in query
self.assertRaises(ValueError, _resolve_uri, "neos://master/db?ca=123")
# credentials with neo:// instead of neos://
self.assertRaises(ValueError, _resolve_uri, "neo://key:zzz@master/db")
# verify zodburi resolver produces expected zconfig # verify zodburi resolver produces expected zconfig
for uri, zconf_ok, dbkw_ok in testv: for uri, env, zconf_ok, dbkw_ok in testv:
zconf_ok = "%import neo.client\n<NEOStorage>\n" + zconf_ok + \ zconf_ok = "%import neo.client\n<NEOStorage>\n" + zconf_ok + \
"</NEOStorage>\n" "</NEOStorage>\n"
zconf, dbkw = _resolve_uri(uri) envsave = os.environ
os.environ = env
try:
zconf, dbkw = _resolve_uri(uri)
finally:
os.environ = envsave
self.assertMultiLineEqual(zconf, zconf_ok) self.assertMultiLineEqual(zconf, zconf_ok)
self.assertEqual(dbkw, dbkw_ok) self.assertEqual(dbkw, dbkw_ok)
......
setup.py
View file @ 95572d6a
...@@ -104,7 +104,7 @@ setup( ...@@ -104,7 +104,7 @@ setup(
'stat_zodb=neo.tests.stat_zodb:main', 'stat_zodb=neo.tests.stat_zodb:main',
], ],
'zodburi.resolvers': [ 'zodburi.resolvers': [
'neo = neo.client.zodburi:resolve_uri [client]', 'neo = neo.client.zodburi:resolve_uri [client]',
'neos = neo.client.zodburi:resolve_uri [client]', 'neos = neo.client.zodburi:resolve_uri [client]',
], ],
}, },
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment
GitLab Nexedi Edition | About GitLab | About Nexedi | 沪ICP备2021021310号-2 | 沪ICP备2021021310号-7