refs #6112 use templates for the vlq functions in the tokudb handlerton

git-svn-id: file:///svn/mysql/tokudb-engine/tokudb-engine@53779 c7de825b-a66e-492c-adef-691d508d4ae1

storage/tokudb/ha_tokudb_update.cc

@@ -4,7 +4,8 @@
 
 // Restrictions:
 //   No triggers
-//   No binary logging
+//   Statement or mixed replication
+//   Does not support row based replication
 //   Primary key must be defined
 //   Simple and compound primary key
 //   Int, char and varchar primary key types
@@ -21,7 +22,6 @@
 
 // Future features:
 //   Support more primary key types
-//   Allow statement based binary logging
 //   Force statement logging for fast updates
 //   Support clustering keys using broadcast updates
 //   Support primary key ranges using multicast messages
@@ -487,17 +487,17 @@ bool ha_tokudb::check_fast_update(THD *thd, List<Item> &fields, List<Item> &valu
 }
 
 static void marshall_varchar_descriptor(tokudb::buffer &b, TABLE *table, KEY_AND_COL_INFO *kc_info, uint key_num) {
-    b.append_uint32('v');
-    b.append_uint32(table->s->null_bytes + kc_info->mcp_info[key_num].fixed_field_size);
+    b.append_ui<uint32_t>('v');
+    b.append_ui<uint32_t>(table->s->null_bytes + kc_info->mcp_info[key_num].fixed_field_size);
     uint32_t var_offset_bytes = kc_info->mcp_info[key_num].len_of_offsets;
-    b.append_uint32(var_offset_bytes);
-    b.append_uint32(var_offset_bytes == 0 ? 0 : kc_info->num_offset_bytes);
+    b.append_ui<uint32_t>(var_offset_bytes);
+    b.append_ui<uint32_t>(var_offset_bytes == 0 ? 0 : kc_info->num_offset_bytes);
 }
 
 static void marshall_blobs_descriptor(tokudb::buffer &b, TABLE *table, KEY_AND_COL_INFO *kc_info) {
-    b.append_uint32('b');
+    b.append_ui<uint32_t>('b');
     uint32_t n = kc_info->num_blobs;
-    b.append_uint32(n);
+    b.append_ui<uint32_t>(n);
     for (uint i = 0; i < n; i++) {
         uint blob_field_index = kc_info->blob_fields[i];
         assert(blob_field_index < table->s->fields);
@@ -615,11 +615,11 @@ static void marshall_update(tokudb::buffer &b, Item *lhs_item, Item *rhs_item, T
     }
 
     // marshall the update fields into the buffer
-    b.append_uint32(update_operation);
-    b.append_uint32(field_type);
-    b.append_uint32(field_null_num);
-    b.append_uint32(offset);
-    b.append_uint32(v_length);
+    b.append_ui<uint32_t>(update_operation);
+    b.append_ui<uint32_t>(field_type);
+    b.append_ui<uint32_t>(field_null_num);
+    b.append_ui<uint32_t>(offset);
+    b.append_ui<uint32_t>(v_length);
     b.append(v_ptr, v_length);
 }
 
@@ -703,7 +703,7 @@ int ha_tokudb::send_update_message(List<Item> &update_fields, List<Item> &update
     }
 
     // append the updates
-    update_message.append_uint32(num_updates);
+    update_message.append_ui<uint32_t>(num_updates);
     
     if (num_varchars > 0 || num_blobs > 0) 
         marshall_varchar_descriptor(update_message, table, &share->kc_info, table->s->primary_key);
@@ -839,7 +839,7 @@ int ha_tokudb::send_upsert_message(THD *thd, List<Item> &update_fields, List<Ite
     update_message.append(&op, sizeof op);
 
     // append the row
-    update_message.append_uint32(row.size);
+    update_message.append_ui<uint32_t>(row.size);
     update_message.append(row.data, row.size);
 
     uint32_t num_updates = update_fields.elements;
@@ -861,7 +861,7 @@ int ha_tokudb::send_upsert_message(THD *thd, List<Item> &update_fields, List<Ite
     }
 
     // append the updates
-    update_message.append_uint32(num_updates);
+    update_message.append_ui<uint32_t>(num_updates);
     
     if (num_varchars > 0 || num_blobs > 0) 
         marshall_varchar_descriptor(update_message, table, &share->kc_info, table->s->primary_key);

storage/tokudb/ha_tokudb_update_fun.cc

@@ -1455,23 +1455,23 @@ static int tokudb_upsert_1_fun(
 
 // Decode and apply a sequence of update operations defined in the extra to the old value and put the result in the new value.
 static void apply_2_updates(tokudb::value_map &vd, tokudb::buffer &new_val, tokudb::buffer &old_val, tokudb::buffer &extra_val) {
-    uint32_t num_updates = extra_val.consume_uint32();
+    uint32_t num_updates; extra_val.consume_ui<uint32_t>(&num_updates);
     for (uint32_t i = 0; i < num_updates; i++) {
-        uint32_t update_operation = extra_val.consume_uint32();
+        uint32_t update_operation; extra_val.consume_ui<uint32_t>(&update_operation);
         if (update_operation == 'v') {
-            uint32_t var_field_offset = extra_val.consume_uint32();
-            uint32_t var_offset_bytes = extra_val.consume_uint32();
-            uint32_t bytes_per_offset = extra_val.consume_uint32();
+            uint32_t var_field_offset; extra_val.consume_ui<uint32_t>(&var_field_offset);
+            uint32_t var_offset_bytes; extra_val.consume_ui<uint32_t>(&var_offset_bytes);
+            uint32_t bytes_per_offset; extra_val.consume_ui<uint32_t>(&bytes_per_offset);
             vd.init_var_fields(var_field_offset, var_offset_bytes, bytes_per_offset);
         } else if (update_operation == 'b') {
-            uint32_t num_blobs = extra_val.consume_uint32();
+            uint32_t num_blobs; extra_val.consume_ui<uint32_t>(&num_blobs);
             uint8_t *blob_lengths = (uint8_t *)extra_val.consume_ptr(num_blobs);
             vd.init_blob_fields(num_blobs, blob_lengths);
         } else {
-            uint32_t field_type = extra_val.consume_uint32();
-            uint32_t field_null_num = extra_val.consume_uint32();
-            uint32_t the_offset = extra_val.consume_uint32();
-            uint32_t extra_val_length = extra_val.consume_uint32();
+            uint32_t field_type; extra_val.consume_ui<uint32_t>(&field_type);
+            uint32_t field_null_num; extra_val.consume_ui<uint32_t>(&field_null_num);
+            uint32_t the_offset; extra_val.consume_ui<uint32_t>(&the_offset);
+            uint32_t extra_val_length; extra_val.consume_ui<uint32_t>(&extra_val_length);
             void *extra_val_ptr = extra_val.consume_ptr(extra_val_length);
 
             switch (field_type) {
@@ -1572,7 +1572,7 @@ static int tokudb_upsert_2_fun(
     extra_val.consume(&op, sizeof op);
     assert(op == UPDATE_OP_UPSERT_2);
 
-    uint32_t insert_length = extra_val.consume_uint32();
+    uint32_t insert_length; extra_val.consume_ui<uint32_t>(&insert_length);
     assert(insert_length < extra_val.limit());
     void *insert_row = extra_val.consume_ptr(insert_length);
 
@@ -1653,3 +1653,8 @@ int tokudb_update_fun(
     }
     return error;
 }
+
+namespace tokudb {
+    template size_t vlq_encode_ui(uint32_t n, void *p, size_t s);
+    template size_t vlq_decode_ui(uint32_t *np, void *p, size_t s);
+};

storage/tokudb/tests/Makefile

@@ -2,7 +2,7 @@ SRCS = $(wildcard *.cc)
 TARGETS = $(patsubst %.cc,%,$(SRCS))
 CHECKS = $(patsubst %,%.check,$(TARGETS))
 CPPFLAGS = -I..
-CXXFLAGS = -g
+CXXFLAGS = -g -fno-implicit-templates
 
 ifeq ($(GCOV),1)
 CXXFLAGS += -fprofile-arcs -ftest-coverage

storage/tokudb/tests/max_test.cc

+// test explicit generation of a simple template function
+
+#include <stdio.h>
+#include <stdlib.h>
+
+template <class T> T my_max(T a, T b) {
+    return a > b ? a : b;
+}
+
+template int my_max(int a, int b);
+
+int main(int argc, char *argv[]) {
+    int a = atoi(argv[1]);
+    int b = atoi(argv[2]);
+    printf("%d %d %d\n", a, b, my_max<int>(a, b));
+    return 0;
+}

storage/tokudb/tests/tokudb_buffer_test.cc

@@ -144,6 +144,51 @@ static void test_replace_null() {
     a.append((void *)"?", 1);
 }
 
+namespace tokudb {
+    template size_t vlq_encode_ui(uint8_t, void *, size_t);
+    template size_t vlq_decode_ui(uint8_t *, void *, size_t);
+    template size_t vlq_encode_ui(uint32_t, void *, size_t);
+    template size_t vlq_decode_ui(uint32_t *, void *, size_t);
+};
+
+static void test_ui8() {
+    tokudb::buffer a;
+    for (uint8_t n = 0; ; n++) {
+        assert(a.append_ui<uint8_t>(n) != 0);
+        if (n == 255)
+            break;
+    }
+    tokudb::buffer b(a.data(), 0, a.size());
+    for (uint8_t n = 0; ; n++) {
+        uint8_t v;
+        if (b.consume_ui<uint8_t>(&v) == 0)
+            break;
+        assert(v == n);
+        if (n == 255)
+            break;
+    }
+    assert(b.size() == b.limit());
+}
+
+static void test_ui32() {
+    tokudb::buffer a;
+    for (uint32_t n = 0; ; n++) {
+        assert(a.append_ui<uint32_t>(n) != 0);
+        if (n == 1<<22)
+            break;
+    }
+    tokudb::buffer b(a.data(), 0, a.size());
+    for (uint32_t n = 0; ; n++) {
+        uint32_t v;
+        if (b.consume_ui<uint32_t>(&v) == 0)
+            break;
+        assert(v == n);
+        if (n == 1<<22)
+            break;
+    }
+    assert(b.size() == b.limit());
+}
+
 int main() {
     test_null();
     test_append();
@@ -153,5 +198,7 @@ int main() {
     test_replace_grow();
     test_replace_shrink();
     test_replace_null();
+    test_ui8();
+    test_ui32();
     return 0;
 }

storage/tokudb/tests/vlq_test.cc

@@ -4,50 +4,151 @@
 #include <assert.h>
 #include <tokudb_vlq.h>
 
-int main(void) {
+namespace tokudb {
+    template size_t vlq_encode_ui(uint32_t n, void *p, size_t s);
+    template size_t vlq_decode_ui(uint32_t *np, void *p, size_t s);
+    template size_t vlq_encode_ui(uint64_t n, void *p, size_t s);
+    template size_t vlq_decode_ui(uint64_t *np, void *p, size_t s);
+};
+
+static void test_vlq_uint32_error(void) {
+    uint32_t n;
+    unsigned char b[5];
+    size_t out_s, in_s;
+
+    out_s = tokudb::vlq_encode_ui<uint32_t>(128, b, 0);
+    assert(out_s == 0);
+    out_s = tokudb::vlq_encode_ui<uint32_t>(128, b, 1);
+    assert(out_s == 0);
+    out_s = tokudb::vlq_encode_ui<uint32_t>(128, b, 2);
+    assert(out_s == 2);
+    in_s = tokudb::vlq_decode_ui<uint32_t>(&n, b, 0);
+    assert(in_s == 0);
+    in_s = tokudb::vlq_decode_ui<uint32_t>(&n, b, 1);
+    assert(in_s == 0);
+    in_s = tokudb::vlq_decode_ui<uint32_t>(&n, b, 2);
+    assert(in_s == 2 && n == 128);
+}
+
+static void test_vlq_uint32(void) {
     uint32_t n;
     unsigned char b[5];
     size_t out_s, in_s;
 
     printf("%u\n", 0);
     for (uint32_t v = 0; v < (1<<7); v++) {
-        out_s = tokudb::vlq_encode_uint32(v, b, sizeof b);
+        out_s = tokudb::vlq_encode_ui<uint32_t>(v, b, sizeof b);
         assert(out_s == 1);
-        in_s = tokudb::vlq_decode_uint32(&n, b, out_s);
+        in_s = tokudb::vlq_decode_ui<uint32_t>(&n, b, out_s);
         assert(in_s == 1 && n == v);
     }
 
     printf("%u\n", 1<<7);
     for (uint32_t v = (1<<7); v < (1<<14); v++) {
-        out_s = tokudb::vlq_encode_uint32(v, b, sizeof b);
+        out_s = tokudb::vlq_encode_ui<uint32_t>(v, b, sizeof b);
         assert(out_s == 2);
-        in_s = tokudb::vlq_decode_uint32(&n, b, out_s);
+        in_s = tokudb::vlq_decode_ui<uint32_t>(&n, b, out_s);
         assert(in_s == 2 && n == v);
     }
 
     printf("%u\n", 1<<14);
     for (uint32_t v = (1<<14); v < (1<<21); v++) {
-        out_s = tokudb::vlq_encode_uint32(v, b, sizeof b);
+        out_s = tokudb::vlq_encode_ui<uint32_t>(v, b, sizeof b);
         assert(out_s == 3);
-        in_s = tokudb::vlq_decode_uint32(&n, b, out_s);
+        in_s = tokudb::vlq_decode_ui<uint32_t>(&n, b, out_s);
         assert(in_s == 3 && n == v);
     }
 
     printf("%u\n", 1<<21);
     for (uint32_t v = (1<<21); v < (1<<28); v++) {
-        out_s = tokudb::vlq_encode_uint32(v, b, sizeof b);
+        out_s = tokudb::vlq_encode_ui<uint32_t>(v, b, sizeof b);
         assert(out_s == 4);
-        in_s = tokudb::vlq_decode_uint32(&n, b, out_s);
+        in_s = tokudb::vlq_decode_ui<uint32_t>(&n, b, out_s);
         assert(in_s == 4 && n == v);
     }
 
     printf("%u\n", 1<<28);
     for (uint32_t v = (1<<28); v != 0; v++) {
-        out_s = tokudb::vlq_encode_uint32(v, b, sizeof b);
+        out_s = tokudb::vlq_encode_ui<uint32_t>(v, b, sizeof b);
         assert(out_s == 5);
-        in_s = tokudb::vlq_decode_uint32(&n, b, out_s);
+        in_s = tokudb::vlq_decode_ui<uint32_t>(&n, b, out_s);
         assert(in_s == 5 && n == v);
     }
+}
+
+static void test_vlq_uint64(void) {
+    uint64_t n;
+    unsigned char b[10];
+    size_t out_s, in_s;
+
+    printf("%u\n", 0);
+    for (uint64_t v = 0; v < (1<<7); v++) {
+        out_s = tokudb::vlq_encode_ui<uint64_t>(v, b, sizeof b);
+        assert(out_s == 1);
+        in_s = tokudb::vlq_decode_ui<uint64_t>(&n, b, out_s);
+        assert(in_s == 1 && n == v);
+    }
 
+    printf("%u\n", 1<<7);
+    for (uint64_t v = (1<<7); v < (1<<14); v++) {
+        out_s = tokudb::vlq_encode_ui<uint64_t>(v, b, sizeof b);
+        assert(out_s == 2);
+        in_s = tokudb::vlq_decode_ui<uint64_t>(&n, b, out_s);
+        assert(in_s == 2 && n == v);
+    }
+
+    printf("%u\n", 1<<14);
+    for (uint64_t v = (1<<14); v < (1<<21); v++) {
+        out_s = tokudb::vlq_encode_ui<uint64_t>(v, b, sizeof b);
+        assert(out_s == 3);
+        in_s = tokudb::vlq_decode_ui<uint64_t>(&n, b, out_s);
+        assert(in_s == 3 && n == v);
+    }
+
+    printf("%u\n", 1<<21);
+    for (uint64_t v = (1<<21); v < (1<<28); v++) {
+        out_s = tokudb::vlq_encode_ui<uint64_t>(v, b, sizeof b);
+        assert(out_s == 4);
+        in_s = tokudb::vlq_decode_ui<uint64_t>(&n, b, out_s);
+        assert(in_s == 4 && n == v);
+    }
+
+    printf("%u\n", 1<<28);
+    for (uint64_t v = (1<<28); v < (1ULL<<35); v++) {
+        out_s = tokudb::vlq_encode_ui<uint64_t>(v, b, sizeof b);
+        assert(out_s == 5);
+        in_s = tokudb::vlq_decode_ui<uint64_t>(&n, b, out_s);
+        assert(in_s == 5 && n == v);
+    }
+}
+
+static void test_80000000(void) {
+    uint64_t n;
+    unsigned char b[10];
+    size_t out_s, in_s;
+    uint64_t v = 0x80000000;
+    out_s = tokudb::vlq_encode_ui<uint64_t>(v, b, sizeof b);
+    assert(out_s == 5);
+    in_s = tokudb::vlq_decode_ui<uint64_t>(&n, b, out_s);
+    assert(in_s == 5 && n == v);
+}
+
+static void test_100000000(void) {
+    uint64_t n;
+    unsigned char b[10];
+    size_t out_s, in_s;
+    uint64_t v = 0x100000000;
+    out_s = tokudb::vlq_encode_ui<uint64_t>(v, b, sizeof b);
+    assert(out_s == 5);
+    in_s = tokudb::vlq_decode_ui<uint64_t>(&n, b, out_s);
+    assert(in_s == 5 && n == v);
+}   
+
+int main(void) {
+    if (1) test_vlq_uint32_error();
+    if (1) test_80000000();
+    if (1) test_100000000();
+    if (1) test_vlq_uint32();
+    if (1) test_vlq_uint64();
     return 0;
 }

storage/tokudb/tokudb_buffer.h

@@ -33,10 +33,14 @@ class buffer {
         memcpy(append_ptr(s), p, s);
     }
 
-    void append_uint32(uint32_t n) {
-        maybe_realloc(5);
-        size_t s = tokudb::vlq_encode_uint32(n, (char *) m_data + m_size, 5);
+    // Append an unsigned int to the buffer.
+    // Returns the number of bytes used to encode the number.
+    // Returns 0 if the number could not be encoded.
+    template<class T> size_t append_ui(T n) {
+        maybe_realloc(10); // 10 bytes is big enough for up to 64 bit number
+        size_t s = tokudb::vlq_encode_ui<T>(n, (char *) m_data + m_size, 10);
         m_size += s;
+        return s;
     }
 
     // Return a pointer to the next location in the buffer where bytes are consumed from.
@@ -53,11 +57,13 @@ class buffer {
         memcpy(p, consume_ptr(s), s);
     }
 
-    uint32_t consume_uint32() {
-        uint32_t n;
-        size_t s = tokudb::vlq_decode_uint32(&n, (char *) m_data + m_size, m_limit - m_size);
+    // Consume an unsigned int from the buffer.
+    // Returns 0 if the unsigned int could not be decoded, probably because the buffer is too short.
+    // Otherwise return the number of bytes consumed, and stuffs the decoded number in *p.
+    template<class T> size_t consume_ui(T *p) {
+        size_t s = tokudb::vlq_decode_ui<T>(p, (char *) m_data + m_size, m_limit - m_size);
         m_size += s;
-        return n;
+        return s;
     }
 
     // Write p_length bytes at an offset in the buffer

storage/tokudb/tokudb_vlq.h

@@ -3,24 +3,36 @@
 
 namespace tokudb {
 
-    static size_t vlq_encode_uint32(uint32_t n, void *p, size_t s) {
+    // Variable length encode an unsigned integer into a buffer with limit s.
+    // Returns the number of bytes used to encode n in the buffer.
+    // Returns 0 if the buffer is too small.
+    template <class T> size_t vlq_encode_ui(T n, void *p, size_t s) {
         unsigned char *pp = (unsigned char *)p;
         size_t i = 0;
         while (n >= 128) {
+            if (i >= s)
+                return 0; // not enough space
             pp[i++] = n%128;
             n = n/128;
         }
+        if (i >= s) 
+            return 0; // not enough space
         pp[i++] = 128+n;
         return i;
     }
 
-    static size_t vlq_decode_uint32(uint32_t *np, void *p, size_t s) {
+    // Variable length decode an unsigned integer from a buffer with limit s.
+    // Returns the number of bytes used to decode the buffer.
+    // Returns 0 if the buffer is too small.
+    template <class T> size_t vlq_decode_ui(T *np, void *p, size_t s) {
         unsigned char *pp = (unsigned char *)p;
-        uint32_t n = 0;
-        uint i = 0;
-        while (i < s) {
-            unsigned char m = pp[i];
-            n |= (m & 127) << 7*i;
+        T n = 0;
+        size_t i = 0;
+        while (1) {
+            if (i >= s)
+                return 0; // not a full decode
+            T m = pp[i];
+            n |= (m & 127) << (7*i);
             i++;
             if ((m & 128) != 0)
                 break;
@@ -28,7 +40,6 @@ namespace tokudb {
         *np = n;
         return i;
     }
-
 }
 
 #endif