go/neo/neonet: Teach Dial to autodetect server encoding

NEO/go server accepts preferred encoding of client from start, but
NEO/py server implements either N or M encoding and disconnects client
silently if handshake is not exactly what is expected.

However we can teach Dial to retry with different preferred options and
this way when connection to a NEO/py server, it will essentially
autodetect which encoding is used and accepted by remote peer.

go/neo/internal/tneonet/tneonet.go

+// Copyright (C) 2021  Nexedi SA and Contributors.
+//                     Kirill Smelkov <kirr@nexedi.com>
+//
+// This program is free software: you can Use, Study, Modify and Redistribute
+// it under the terms of the GNU General Public License version 3, or (at your
+// option) any later version, as published by the Free Software Foundation.
+//
+// You can also Link and Combine this program with other software covered by
+// the terms of any of the Free Software licenses or any of the Open Source
+// Initiative approved licenses and Convey the resulting work. Corresponding
+// source of such a combination shall include the source code for all other
+// software used.
+//
+// This program is distributed WITHOUT ANY WARRANTY; without even the implied
+// warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
+//
+// See COPYING file for full licensing terms.
+// See https://www.nexedi.com/licensing for rationale and options.
+
+// Package tneonet provides testing hooks for neonet package.
+package tneonet
+
+import (
+	"lab.nexedi.com/kirr/neo/go/neo/proto"
+)
+
+var DialEncTryOrder *[]proto.Encoding // = &neonet.dialEncTryOrder

go/neo/neonet/newlink.go

@@ -23,6 +23,7 @@ package neonet
 import (
 	"bytes"
 	"context"
+	"errors"
 	"fmt"
 	"io"
 	"net"
@@ -33,6 +34,7 @@ import (
 	"lab.nexedi.com/kirr/go123/xnet"
 	"lab.nexedi.com/kirr/neo/go/internal/xcontext"
 	"lab.nexedi.com/kirr/neo/go/internal/xio"
+	"lab.nexedi.com/kirr/neo/go/neo/internal/tneonet"
 	"lab.nexedi.com/kirr/neo/go/neo/proto"
 )
 
@@ -43,7 +45,7 @@ import (
 // do not have such uses.
 
 func _HandshakeClient(ctx context.Context, conn net.Conn) (*NodeLink, error) {
-	return handshakeClient(ctx, conn, proto.Version, proto.Encoding('N'))
+	return handshakeClient(ctx, conn, proto.Version, dialEncTryOrder[0])
 }
 
 func _HandshakeServer(ctx context.Context, conn net.Conn) (*NodeLink, error) {
@@ -273,15 +275,42 @@ func rxHello(errctx string, rx *xbufReader) (enc proto.Encoding, version uint32,
 
 // ---- Dial & Listen at NodeLink level ----
 
+// dialEncTryOrder is the order of trials for encoding when establishing a NEO link from client side.
+//
+// NEO/go server autodetects client preferred encoding and adjusts to that, but
+// NEO/py rejects connections if client encoding does not exactly match server.
+//
+// To autodetect what NEO/py server uses DialLink retries dial + handshake with
+// client-preferred encodings in the order specified by dialEncTryOrder.
+//
+// NOTE tests change this to verify that autodetection of NEO/py server
+// encoding actually works.
+var dialEncTryOrder = []proto.Encoding{'N', 'M'}
+func init() {
+	tneonet.DialEncTryOrder = &dialEncTryOrder
+}
+
 // DialLink connects to address on given network, performs NEO protocol
 // handshake and wraps the connection as NodeLink.
-func DialLink(ctx context.Context, net xnet.Networker, addr string) (*NodeLink, error) {
-	peerConn, err := net.Dial(ctx, addr)
-	if err != nil {
-		return nil, err
+func DialLink(ctx context.Context, net xnet.Networker, addr string) (link *NodeLink, err error) {
+	for _, enc := range dialEncTryOrder {
+		peerConn, err := net.Dial(ctx, addr)
+		if err != nil {
+			return nil, err
+		}
+
+		link, err = handshakeClient(ctx, peerConn, proto.Version, enc)
+
+		// NEO/py closes connection if it sees unexpected magic, version, etc.
+		// -> in such case retry with next encoding trying to autodetect and match server.
+		// -> stop trying on success, or on any other error.
+		if err == nil || !errors.Is(err, io.ErrUnexpectedEOF) {
+			break
+		}
 	}
 
-	return _HandshakeClient(ctx, peerConn)
+	// either link is established or the error indicates we should not retry anymore
+	return link, err
 }
 
 // ListenLink starts listening on laddr for incoming connections and wraps them as NodeLink.