decoder: Fix panic on dict with non-comparable keys

Even though we tried to catch whether dict keys are ok to be used via
reflect.TypeOf(key).Comparable() (see da5f0342 "decoder: Fix crashes
found by fuzzer (#32)"), that turned out to be not enough. For example
if key is a struct, e.g. of the following type

	type Ref struct {
		Pid interface{}
	}

it will be comparable. But the comparision, depending on dynamic .Pid
type, might panic. This is what was actually cauht by fuzz-testing
recently:

	https://github.com/kisielk/og-rek/issues/50 (second part of the report)

So instead of recursively walking a key type and checking each subfield
with reflect.TypeOf().Comparable(), switch for using panic/recover for
detecting the "unhashable key" situation.

This slows down decoding a bit (only cumulative figure for all-test-vectors decoding):

	name          old time/op    new time/op    delta
	DecodeLong-4     361ns ± 0%     362ns ± 0%    ~     (p=0.238 n=5+4)
	Decode-4        93.2µs ± 0%    95.6µs ± 0%  +2.54%  (p=0.008 n=5+5)
	Encode-4        16.5µs ± 0%    16.6µs ± 0%    ~     (p=0.841 n=5+5)

but that is the price of correctness. And with manually recursively walking key
type I doubt it would be faster.

The defer overhead should be less once https://github.com/golang/go/issues/14939 is fixed.

Updates: https://github.com/kisielk/og-rek/issues/30

ogorek.go

@@ -12,7 +12,6 @@ import (
 	"io"
 	"math"
 	"math/big"
-	"reflect"
 	"strconv"
 )
 
@@ -809,6 +808,37 @@ func (d *Decoder) global() error {
 	return nil
 }
 
+// mapTryAssign tries to do `m[key] = value`.
+//
+// It checks whether key is of appropriate type, and if yes - succeeds.
+// If key is not appropriate - the map stays unchanged and false is returned.
+func mapTryAssign(m map[interface{}]interface{}, key, value interface{}) (ok bool) {
+	// use panic/recover to detect inappropriate keys.
+	//
+	// We could try to use reflect.TypeOf(key).Comparable() instead, but that
+	// is not generally enough: with Comparable, key type structure has to
+	// be manually walked recursively and each subfield checked for
+	// comparability. -> panic/recover is simpler to use instead.
+	//
+	// See https://github.com/kisielk/og-rek/issues/30#issuecomment-423803200
+	// for details.
+	defer func() {
+		// on invalid dynamic key type runtime panics like below:
+		//
+		//	`panic: runtime error: hash of unhashable type []interface {}`
+		//
+		// we don't try to detect the exact message as mapTryAssign does
+		// only 1 operation.
+		if r := recover(); r != nil {
+			ok = false
+		}
+	}()
+
+	m[key] = value
+	ok = true
+	return
+}
+
 func (d *Decoder) loadDict() error {
 	k, err := d.marker()
 	if err != nil {
@@ -822,10 +852,9 @@ func (d *Decoder) loadDict() error {
 	}
 	for i := 0; i < len(items); i += 2 {
 		key := items[i]
-		if !reflect.TypeOf(key).Comparable() {
+		if !mapTryAssign(m, key, items[i+1]) {
 			return fmt.Errorf("pickle: loadDict: invalid key type %T", key)
 		}
-		m[key] = items[i+1]
 	}
 	d.stack = append(d.stack[:k], m)
 	return nil
@@ -1018,10 +1047,9 @@ func (d *Decoder) loadSetItem() error {
 	m := d.stack[len(d.stack)-1]
 	switch m := m.(type) {
 	case map[interface{}]interface{}:
-		if !reflect.TypeOf(k).Comparable() {
+		if !mapTryAssign(m, k, v) {
 			return fmt.Errorf("pickle: loadSetItem: invalid key type %T", k)
 		}
-		m[k] = v
 	default:
 		return fmt.Errorf("pickle: loadSetItem: expected a map, got %T", m)
 	}
@@ -1045,10 +1073,9 @@ func (d *Decoder) loadSetItems() error {
 		}
 		for i := k + 1; i < len(d.stack); i += 2 {
 			key := d.stack[i]
-			if !reflect.TypeOf(key).Comparable() {
+			if !mapTryAssign(m, key, d.stack[i+1]) {
 				return fmt.Errorf("pickle: loadSetItems: invalid key type %T", key)
 			}
-			m[d.stack[i]] = d.stack[i+1]
 		}
 		d.stack = append(d.stack[:k-1], m)
 	default:

ogorek_test.go

@@ -758,6 +758,9 @@ func TestFuzzCrashers(t *testing.T) {
 		"(c\n\nc\n\n\x85Rd",
 		"}(U\x040000u",
 		"(\x88d",
+		"(]QNd.",       // PersID([])      -> dict
+		"}]QNs.",       // PersID([])      -> setitem
+		"}(]QNI1\nNu.", // PersID([]) ...  -> setitems
 	}
 
 	for _, c := range crashers {