Merge pull request #35 from navytux/fix1

decoder: Small speedups

ogorek.go

@@ -116,16 +116,19 @@ type Decoder struct {
 	// a reusable buffer that can be used by the various decoding functions
 	// functions using this should call buf.Reset to clear the old contents
 	buf bytes.Buffer
+
+	// reusable buffer for readLine
+	line  []byte
 }
 
 // NewDecoder constructs a new Decoder which will decode the pickle stream in r.
-func NewDecoder(r io.Reader) Decoder {
+func NewDecoder(r io.Reader) *Decoder {
 	reader := bufio.NewReader(r)
-	return Decoder{r: reader, stack: make([]interface{}, 0), memo: make(map[string]interface{})}
+	return &Decoder{r: reader, stack: make([]interface{}, 0), memo: make(map[string]interface{})}
 }
 
 // Decode decodes the pickle stream and returns the result or an error.
-func (d Decoder) Decode() (interface{}, error) {
+func (d *Decoder) Decode() (interface{}, error) {
 
 	insn := 0
 loop:
@@ -265,21 +268,23 @@ loop:
 	return d.pop()
 }
 
+// readLine reads next line from pickle stream
+// returned line is valid only till next call to readLine
 func (d *Decoder) readLine() ([]byte, error) {
 	var (
-		line     []byte
 		data     []byte
 		isPrefix = true
 		err      error
 	)
+	d.line = d.line[:0]
 	for isPrefix {
 		data, isPrefix, err = d.r.ReadLine()
 		if err != nil {
-			return line, err
+			return d.line, err
 		}
-		line = append(line, data...)
+		d.line = append(d.line, data...)
 	}
-	return line, nil
+	return d.line, nil
 }
 
 // Push a marker
@@ -536,6 +541,7 @@ func (d *Decoder) loadBinString() error {
 	v := binary.LittleEndian.Uint32(b[:])
 
 	d.buf.Reset()
+	d.buf.Grow(int(v))
 	_, err = io.CopyN(&d.buf, d.r, int64(v))
 	if err != nil {
 		return err
@@ -551,6 +557,7 @@ func (d *Decoder) loadShortBinString() error {
 	}
 
 	d.buf.Reset()
+	d.buf.Grow(int(b))
 	_, err = io.CopyN(&d.buf, d.r, int64(b))
 	if err != nil {
 		return err
@@ -568,6 +575,7 @@ func (d *Decoder) loadUnicode() error {
 	sline := string(line)
 
 	d.buf.Reset()
+	d.buf.Grow(len(line)) // approximation
 
 	for len(sline) > 0 {
 		var r rune
@@ -643,11 +651,13 @@ func (d *Decoder) global() error {
 	if err != nil {
 		return err
 	}
+	smodule := string(module)
 	name, err := d.readLine()
 	if err != nil {
 		return err
 	}
-	d.stack = append(d.stack, Class{Module: string(module), Name: string(name)})
+	sname := string(name)
+	d.stack = append(d.stack, Class{Module: smodule, Name: sname})
 	return nil
 }
 
@@ -925,6 +935,7 @@ func (d *Decoder) loadShortBinUnicode() error {
 	}
 
 	d.buf.Reset()
+	d.buf.Grow(int(b))
 	_, err = io.CopyN(&d.buf, d.r, int64(b))
 	if err != nil {
 		return err

ogorek_test.go

@@ -37,12 +37,11 @@ var graphitePickle1, _ = hex.DecodeString("80025d71017d710228550676616c756573710
 var graphitePickle2, _ = hex.DecodeString("286c70300a286470310a53277374617274270a70320a49313338333738323430300a73532773746570270a70330a4938363430300a735327656e64270a70340a49313338353136343830300a73532776616c756573270a70350a286c70360a463437332e300a61463439372e300a61463534302e300a6146313439372e300a6146313830382e300a6146313839302e300a6146323031332e300a6146313832312e300a6146313834372e300a6146323137362e300a6146323135362e300a6146313235302e300a6146323035352e300a6146313537302e300a614e614e617353276e616d65270a70370a5327757365722e6c6f67696e2e617265612e6d616368696e652e6d65747269632e6d696e757465270a70380a73612e")
 var graphitePickel3, _ = hex.DecodeString("286c70310a286470320a5327696e74657276616c73270a70330a286c70340a7353276d65747269635f70617468270a70350a5327636172626f6e2e6167656e7473270a70360a73532769734c656166270a70370a4930300a7361286470380a67330a286c70390a7367350a5327636172626f6e2e61676772656761746f72270a7031300a7367370a4930300a736128647031310a67330a286c7031320a7367350a5327636172626f6e2e72656c617973270a7031330a7367370a4930300a73612e")
 
-func TestDecode(t *testing.T) {
-	tests := []struct {
+var tests = []struct {
 	name     string
 	input    string
 	expected interface{}
-	}{
+}{
 	{"int", "I5\n.", int64(5)},
 	{"float", "F1.23\n.", float64(1.23)},
 	{"long", "L12321231232131231231L\n.", bigInt("12321231232131231231")},
@@ -69,7 +68,9 @@ func TestDecode(t *testing.T) {
 	{"too long line", "V28,34,30,55,100,130,87,169,194,202,232,252,267,274,286,315,308,221,358,368,401,406,434,452,475,422,497,530,517,559,400,418,571,578,599,600,625,630,635,647,220,715,736,760,705,785,794,495,808,852,861,863,869,875,890,893,896,922,812,980,1074,1087,1145,1153,1163,1171,445,1195,1203,1242,1255,1274,52,1287,1319,636,1160,1339,1345,1353,1369,1391,1396,1405,1221,1410,1431,1451,1460,1470,1472,1492,1517,1528,419,1530,1532,1535,1573,1547,1574,1437,1594,1595,847,1551,983,1637,1647,1666,1672,1691,1726,1515,1731,1739,1741,1723,1776,1685,505,1624,1436,1890,728,1910,1931,1544,2013,2025,2030,2043,2069,1162,2129,2160,2199,2210,1911,2246,804,2276,1673,2299,2315,2322,2328,2355,2376,2405,1159,2425,2430,2452,1804,2442,2567,2577,1167,2611,2534,1879,2623,2682,2699,2652,2742,2754,2774,2782,2795,2431,2821,2751,2850,2090,513,2898,592,2932,2933,1555,2969,3003,3007,3010,2595,3064,3087,3105,3106,3110,151,3129,3132,304,3173,3205,3233,3245,3279,3302,3307,714,316,3331,3347,3360,3375,3380,3442,2620,3482,3493,3504,3516,3517,3518,3533,3511,2681,3530,3601,3606,3615,1210,3633,3651,3688,3690,3781,1907,3839,3840,3847,3867,3816,3899,3924,2345,3912,3966,982,4040,4056,4076,4084,4105,2649,4171,3873,1415,3567,4188,4221,4227,4231,2279,4250,4253,770,894,4343,4356,4289,4404,4438,2572,3124,4334,2114,3953,4522,4537,4561,4571,641,4629,4640,4664,4687,4702,4709,4740,4605,4746,4768,3856,3980,4814,2984,4895,4908,1249,4944,4947,4979,4988,4995,32,4066,5043,4956,5069,5072,5076,5084,5085,5137,4262,5152,479,5156,3114,1277,5183,5186,1825,5106,5216,963,5239,5252,5218,5284,1980,1972,5352,5364,5294,5379,5387,5391,5397,5419,5434,5468,5471,3350,5510,5522,5525,5538,5554,5573,5597,5610,5615,5624,842,2851,5641,5655,5656,5658,5678,5682,5696,5699,5709,5728,5753,851,5805,3528,5822,801,5855,2929,5871,5899,5918,5925,5927,5931,5935,5939,5958,778,5971,5980,5300,6009,6023,6030,6032,6016,6110,5009,6155,6197,1760,6253,6267,4886,5608,6289,6308,6311,6321,6316,6333,6244,6070,6349,6353,6186,6357,6366,6386,6387,6389,6399,6411,6421,6432,6437,6465,6302,6493,5602,6511,6529,6536,6170,6557,6561,6577,6581,6590,5290,5649,6231,6275,6635,6651,6652,5929,6692,6693,6695,6705,6711,6723,6738,6752,6753,3629,2975,6790,5845,338,6814,6826,6478,6860,6872,6882,880,356,6897,4102,6910,6611,1030,6934,6936,6987,6984,6999,827,6902,7027,7049,7051,4628,7084,7083,7071,7102,7137,5867,7152,6048,2410,3896,7168,7177,7224,6606,7233,1793,7261,7284,7290,7292,5212,7315,6964,3238,355,1969,4256,448,7325,908,2824,2981,3193,3363,3613,5325,6388,2247,1348,72,131,5414,7285,7343,7349,7362,7372,7381,7410,7418,7443,5512,7470,7487,7497,7516,7277,2622,2863,945,4344,3774,1024,2272,7523,4476,256,5643,3164,7539,7540,7489,1932,7559,7575,7602,7605,7609,7608,7619,7204,7652,7663,6907,7672,7654,7674,7687,7718,7745,1202,4030,7797,7801,7799,2924,7871,7873,7900,7907,7911,7912,7917,7923,7935,8007,8017,7636,8084,8087,3686,8114,8153,8158,8171,8175,8182,8205,8222,8225,8229,8232,8234,8244,8247,7256,8279,6929,8285,7040,8328,707,6773,7949,8468,5759,6344,8509,1635\n.", "28,34,30,55,100,130,87,169,194,202,232,252,267,274,286,315,308,221,358,368,401,406,434,452,475,422,497,530,517,559,400,418,571,578,599,600,625,630,635,647,220,715,736,760,705,785,794,495,808,852,861,863,869,875,890,893,896,922,812,980,1074,1087,1145,1153,1163,1171,445,1195,1203,1242,1255,1274,52,1287,1319,636,1160,1339,1345,1353,1369,1391,1396,1405,1221,1410,1431,1451,1460,1470,1472,1492,1517,1528,419,1530,1532,1535,1573,1547,1574,1437,1594,1595,847,1551,983,1637,1647,1666,1672,1691,1726,1515,1731,1739,1741,1723,1776,1685,505,1624,1436,1890,728,1910,1931,1544,2013,2025,2030,2043,2069,1162,2129,2160,2199,2210,1911,2246,804,2276,1673,2299,2315,2322,2328,2355,2376,2405,1159,2425,2430,2452,1804,2442,2567,2577,1167,2611,2534,1879,2623,2682,2699,2652,2742,2754,2774,2782,2795,2431,2821,2751,2850,2090,513,2898,592,2932,2933,1555,2969,3003,3007,3010,2595,3064,3087,3105,3106,3110,151,3129,3132,304,3173,3205,3233,3245,3279,3302,3307,714,316,3331,3347,3360,3375,3380,3442,2620,3482,3493,3504,3516,3517,3518,3533,3511,2681,3530,3601,3606,3615,1210,3633,3651,3688,3690,3781,1907,3839,3840,3847,3867,3816,3899,3924,2345,3912,3966,982,4040,4056,4076,4084,4105,2649,4171,3873,1415,3567,4188,4221,4227,4231,2279,4250,4253,770,894,4343,4356,4289,4404,4438,2572,3124,4334,2114,3953,4522,4537,4561,4571,641,4629,4640,4664,4687,4702,4709,4740,4605,4746,4768,3856,3980,4814,2984,4895,4908,1249,4944,4947,4979,4988,4995,32,4066,5043,4956,5069,5072,5076,5084,5085,5137,4262,5152,479,5156,3114,1277,5183,5186,1825,5106,5216,963,5239,5252,5218,5284,1980,1972,5352,5364,5294,5379,5387,5391,5397,5419,5434,5468,5471,3350,5510,5522,5525,5538,5554,5573,5597,5610,5615,5624,842,2851,5641,5655,5656,5658,5678,5682,5696,5699,5709,5728,5753,851,5805,3528,5822,801,5855,2929,5871,5899,5918,5925,5927,5931,5935,5939,5958,778,5971,5980,5300,6009,6023,6030,6032,6016,6110,5009,6155,6197,1760,6253,6267,4886,5608,6289,6308,6311,6321,6316,6333,6244,6070,6349,6353,6186,6357,6366,6386,6387,6389,6399,6411,6421,6432,6437,6465,6302,6493,5602,6511,6529,6536,6170,6557,6561,6577,6581,6590,5290,5649,6231,6275,6635,6651,6652,5929,6692,6693,6695,6705,6711,6723,6738,6752,6753,3629,2975,6790,5845,338,6814,6826,6478,6860,6872,6882,880,356,6897,4102,6910,6611,1030,6934,6936,6987,6984,6999,827,6902,7027,7049,7051,4628,7084,7083,7071,7102,7137,5867,7152,6048,2410,3896,7168,7177,7224,6606,7233,1793,7261,7284,7290,7292,5212,7315,6964,3238,355,1969,4256,448,7325,908,2824,2981,3193,3363,3613,5325,6388,2247,1348,72,131,5414,7285,7343,7349,7362,7372,7381,7410,7418,7443,5512,7470,7487,7497,7516,7277,2622,2863,945,4344,3774,1024,2272,7523,4476,256,5643,3164,7539,7540,7489,1932,7559,7575,7602,7605,7609,7608,7619,7204,7652,7663,6907,7672,7654,7674,7687,7718,7745,1202,4030,7797,7801,7799,2924,7871,7873,7900,7907,7911,7912,7917,7923,7935,8007,8017,7636,8084,8087,3686,8114,8153,8158,8171,8175,8182,8205,8222,8225,8229,8232,8234,8244,8247,7256,8279,6929,8285,7040,8328,707,6773,7949,8468,5759,6344,8509,1635"},
 	{"FRAME Opcode and int", "\x95\x00\x00\x00\x00\x00\x00\x00\x00I5\n.", int64(5)},
 	{"SHORTBINUNICODE opcode", "\x8c\t\xe6\x97\xa5\xe6\x9c\xac\xe8\xaa\x9e\x94.", "日本語"},
-	}
+}
+
+func TestDecode(t *testing.T) {
 	for _, test := range tests {
 		// decode(input) -> expected
 		buf := bytes.NewBufferString(test.input)
@@ -322,3 +323,55 @@ func TestFuzzCrashers(t *testing.T) {
 		dec.Decode()
 	}
 }
+
+
+func BenchmarkDecode(b *testing.B) {
+	// prepare one large pickle stream from all test pickles
+	input := make([]byte, 0)
+	for _, test := range tests {
+		input = append(input, test.input...)
+	}
+
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		buf := bytes.NewBuffer(input)
+		dec := NewDecoder(buf)
+
+		j := 0
+		for ; ; j++ {
+			_, err := dec.Decode()
+			if err != nil {
+				if err == io.EOF {
+					break
+				}
+				b.Fatal(err)
+			}
+		}
+
+		if j != len(tests) {
+			b.Fatalf("unexpected # of decode steps: got %v  ; want %v", j, len(tests))
+		}
+	}
+}
+
+func BenchmarkEncode(b *testing.B) {
+	// prepare one large slice from all test vector values
+	input := make([]interface{}, 0)
+	approxOutSize := 0
+	for _, test := range tests {
+		input = append(input, test.expected)
+		approxOutSize += len(test.input)
+	}
+
+	buf := bytes.NewBuffer(make([]byte, approxOutSize))
+
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		buf.Reset()
+		enc := NewEncoder(buf)
+		err := enc.Encode(input)
+		if err != nil {
+			b.Fatal(err)
+		}
+	}
+}