#5023 always compile in the update callback in the handlerton (even for mysql...

#5023 always compile in the update callback in the handlerton (even for mysql 5.5 without hcad) refs[t:5023]

git-svn-id: file:///svn/mysql/tokudb-engine/tokudb-engine@44263 c7de825b-a66e-492c-adef-691d508d4ae1

storage/tokudb/ha_tokudb.cc

@@ -8000,6 +8000,7 @@ void ha_tokudb::set_dup_value_for_pk(DBT* key) {
 #include "ha_tokudb_admin.cc"
 
 // alter table code for various mysql distros
+#include "ha_tokudb_update_fun.cc"
 #include "ha_tokudb_alter_51.cc"
 #include "ha_tokudb_alter_55.cc"
 #include "ha_tokudb_alter_56.cc"

storage/tokudb/ha_tokudb.h

@@ -13,11 +13,7 @@
 #define TOKU_INCLUDE_ROW_TYPE_COMPRESSION 1
 #endif
 
-#if defined(HA_GENERAL_ONLINE) || (50600 <= MYSQL_VERSION_ID && MYSQL_VERSION_ID <= 50699)
 #define TOKU_INCLUDE_UPDATE_FUN 1
-#else
-#define TOKU_INCLUDE_UPDATE_FUN 0
-#endif
 
 #if !defined(HA_CLUSTERING)
 #define HA_CLUSTERING 0

storage/tokudb/ha_tokudb_update_fun.cc

+#if TOKU_INCLUDE_UPDATE_FUN
+
+#define UP_COL_ADD_OR_DROP 0
+
+#define COL_DROP 0xaa
+#define COL_ADD 0xbb
+
+#define COL_FIXED 0xcc
+#define COL_VAR 0xdd
+#define COL_BLOB 0xee
+
+
+
+#define STATIC_ROW_MUTATOR_SIZE 1+8+2+8+8+8
+
+/*
+how much space do I need for the mutators?
+static stuff first:
+1 - UP_COL_ADD_OR_DROP
+8 - old null, new null
+2 - old num_offset, new num_offset
+8 - old fixed_field size, new fixed_field_size
+8 - old and new length of offsets
+8 - old and new starting null bit position
+TOTAL: 27
+
+dynamic stuff:
+4 - number of columns
+for each column:
+1 - add or drop
+1 - is nullable
+4 - if nullable, position
+1 - if add, whether default is null or not
+1 - if fixed, var, or not
+ for fixed, entire default
+ for var, 4 bytes length, then entire default
+ for blob, nothing
+So, an upperbound is 4 + num_fields(12) + all default stuff
+
+static blob stuff:
+4 - num blobs
+1 byte for each num blobs in old table
+So, an upperbound is 4 + kc_info->num_blobs
+
+dynamic blob stuff:
+for each blob added:
+1 - state if we are adding or dropping
+4 - blob index
+if add, 1 len bytes
+ at most, 4 0's
+So, upperbound is num_blobs(1+4+1+4) = num_columns*10
+*/
+
+//
+// checks whether the bit at index pos in data is set or not
+//
+static inline bool 
+is_overall_null_position_set(uchar* data, u_int32_t pos) {
+    u_int32_t offset = pos/8;
+    uchar remainder = pos%8; 
+    uchar null_bit = 1<<remainder;
+    return ((data[offset] & null_bit) != 0);
+}
+
+//
+// sets the bit at index pos in data to 1 if is_null, 0 otherwise
+// 
+static inline void 
+set_overall_null_position(uchar* data, u_int32_t pos, bool is_null) {
+    u_int32_t offset = pos/8;
+    uchar remainder = pos%8;
+    uchar null_bit = 1<<remainder;
+    if (is_null) {
+        data[offset] |= null_bit;
+    }
+    else {
+        data[offset] &= ~null_bit;
+    }
+}
+
+static inline void 
+copy_null_bits(
+    u_int32_t start_old_pos,
+    u_int32_t start_new_pos,
+    u_int32_t num_bits,
+    uchar* old_null_bytes,
+    uchar* new_null_bytes
+    ) 
+{
+    for (u_int32_t i = 0; i < num_bits; i++) {
+        u_int32_t curr_old_pos = i + start_old_pos;
+        u_int32_t curr_new_pos = i + start_new_pos;
+        // copy over old null bytes
+        if (is_overall_null_position_set(old_null_bytes,curr_old_pos)) {
+            set_overall_null_position(new_null_bytes,curr_new_pos,true);
+        }
+        else {
+            set_overall_null_position(new_null_bytes,curr_new_pos,false);
+        }
+    }
+}
+
+static inline void 
+copy_var_fields(
+    u_int32_t start_old_num_var_field, //index of var fields that we should start writing
+    u_int32_t num_var_fields, // number of var fields to copy
+    uchar* old_var_field_offset_ptr, //static ptr to where offset bytes begin in old row
+    uchar old_num_offset_bytes, //number of offset bytes used in old row
+    uchar* start_new_var_field_data_ptr, // where the new var data should be written
+    uchar* start_new_var_field_offset_ptr, // where the new var offsets should be written
+    uchar* new_var_field_data_ptr, // pointer to beginning of var fields in new row
+    uchar* old_var_field_data_ptr, // pointer to beginning of var fields in old row
+    u_int32_t new_num_offset_bytes, // number of offset bytes used in new row
+    u_int32_t* num_data_bytes_written,
+    u_int32_t* num_offset_bytes_written
+    ) 
+{
+    uchar* curr_new_var_field_data_ptr = start_new_var_field_data_ptr;
+    uchar* curr_new_var_field_offset_ptr = start_new_var_field_offset_ptr;
+    for (u_int32_t i = 0; i < num_var_fields; i++) {
+        u_int32_t field_len;
+        u_int32_t start_read_offset;
+        u_int32_t curr_old = i + start_old_num_var_field;
+        uchar* data_to_copy = NULL;
+        // get the length and pointer to data that needs to be copied
+        get_var_field_info(
+            &field_len, 
+            &start_read_offset, 
+            curr_old, 
+            old_var_field_offset_ptr, 
+            old_num_offset_bytes
+            );
+        data_to_copy = old_var_field_data_ptr + start_read_offset;
+        // now need to copy field_len bytes starting from data_to_copy
+        curr_new_var_field_data_ptr = write_var_field(
+            curr_new_var_field_offset_ptr,
+            curr_new_var_field_data_ptr,
+            new_var_field_data_ptr,
+            data_to_copy,
+            field_len,
+            new_num_offset_bytes
+            );
+        curr_new_var_field_offset_ptr += new_num_offset_bytes;
+    }
+    *num_data_bytes_written = (u_int32_t)(curr_new_var_field_data_ptr - start_new_var_field_data_ptr);
+    *num_offset_bytes_written = (u_int32_t)(curr_new_var_field_offset_ptr - start_new_var_field_offset_ptr);
+}
+
+static inline u_int32_t 
+copy_toku_blob(uchar* to_ptr, uchar* from_ptr, u_int32_t len_bytes, bool skip) {
+    u_int32_t length = 0;
+    if (!skip) {
+        memcpy(to_ptr, from_ptr, len_bytes);
+    }
+    length = get_blob_field_len(from_ptr,len_bytes);
+    if (!skip) {
+        memcpy(to_ptr + len_bytes, from_ptr + len_bytes, length);
+    }
+    return (length + len_bytes);
+}
+
+int 
+tokudb_update_fun(
+    DB* db,
+    const DBT *key,
+    const DBT *old_val, 
+    const DBT *extra,
+    void (*set_val)(const DBT *new_val, void *set_extra),
+    void *set_extra
+    ) 
+{
+    u_int32_t max_num_bytes;
+    u_int32_t num_columns;
+    DBT new_val;
+    u_int32_t num_bytes_left;
+    u_int32_t num_var_fields_to_copy;
+    u_int32_t num_data_bytes_written = 0;
+    u_int32_t num_offset_bytes_written = 0;
+    int error;
+    memset(&new_val, 0, sizeof(DBT));
+    uchar operation;
+    uchar* new_val_data = NULL;
+    uchar* extra_pos = NULL;
+    uchar* extra_pos_start = NULL;
+    //
+    // info for pointers into rows
+    //
+    u_int32_t old_num_null_bytes;
+    u_int32_t new_num_null_bytes;
+    uchar old_num_offset_bytes;
+    uchar new_num_offset_bytes;
+    u_int32_t old_fixed_field_size;
+    u_int32_t new_fixed_field_size;
+    u_int32_t old_len_of_offsets;
+    u_int32_t new_len_of_offsets;
+
+    uchar* old_fixed_field_ptr = NULL;
+    uchar* new_fixed_field_ptr = NULL;
+    u_int32_t curr_old_fixed_offset;
+    u_int32_t curr_new_fixed_offset;
+
+    uchar* old_null_bytes = NULL;
+    uchar* new_null_bytes = NULL;
+    u_int32_t curr_old_null_pos;
+    u_int32_t curr_new_null_pos;    
+    u_int32_t old_null_bits_left;
+    u_int32_t new_null_bits_left;
+    u_int32_t overall_null_bits_left;
+
+    u_int32_t old_num_var_fields;
+    u_int32_t new_num_var_fields;
+    u_int32_t curr_old_num_var_field;
+    u_int32_t curr_new_num_var_field;
+    uchar* old_var_field_offset_ptr = NULL;
+    uchar* new_var_field_offset_ptr = NULL;
+    uchar* curr_new_var_field_offset_ptr = NULL;
+    uchar* old_var_field_data_ptr = NULL;
+    uchar* new_var_field_data_ptr = NULL;
+    uchar* curr_new_var_field_data_ptr = NULL;
+
+    u_int32_t start_blob_offset;
+    uchar* start_blob_ptr;
+    u_int32_t num_blob_bytes;
+
+    // came across a delete, nothing to update
+    if (old_val == NULL) {
+        error = 0;
+        goto cleanup;
+    }
+
+    extra_pos_start = (uchar *)extra->data;
+    extra_pos = (uchar *)extra->data;
+
+    operation = extra_pos[0];
+    extra_pos++;
+    assert(operation == UP_COL_ADD_OR_DROP);
+
+    memcpy(&old_num_null_bytes, extra_pos, sizeof(u_int32_t));
+    extra_pos += sizeof(u_int32_t);
+    memcpy(&new_num_null_bytes, extra_pos, sizeof(u_int32_t));
+    extra_pos += sizeof(u_int32_t);
+
+    old_num_offset_bytes = extra_pos[0];
+    extra_pos++;
+    new_num_offset_bytes = extra_pos[0];
+    extra_pos++;
+
+    memcpy(&old_fixed_field_size, extra_pos, sizeof(u_int32_t));
+    extra_pos += sizeof(u_int32_t);
+    memcpy(&new_fixed_field_size, extra_pos, sizeof(u_int32_t));
+    extra_pos += sizeof(u_int32_t);
+
+    memcpy(&old_len_of_offsets, extra_pos, sizeof(u_int32_t));
+    extra_pos += sizeof(u_int32_t);
+    memcpy(&new_len_of_offsets, extra_pos, sizeof(u_int32_t));
+    extra_pos += sizeof(u_int32_t);
+
+    max_num_bytes = old_val->size + extra->size + new_len_of_offsets + new_fixed_field_size;
+    new_val_data = (uchar *)my_malloc(
+        max_num_bytes, 
+        MYF(MY_FAE)
+        );
+    if (new_val_data == NULL) { goto cleanup; }
+
+    old_fixed_field_ptr = (uchar *) old_val->data;
+    old_fixed_field_ptr += old_num_null_bytes;
+    new_fixed_field_ptr = new_val_data + new_num_null_bytes;
+    curr_old_fixed_offset = 0;
+    curr_new_fixed_offset = 0;
+
+    old_num_var_fields = old_len_of_offsets/old_num_offset_bytes;
+    new_num_var_fields = new_len_of_offsets/new_num_offset_bytes;
+    // following fields will change as we write the variable data
+    old_var_field_offset_ptr = old_fixed_field_ptr + old_fixed_field_size;
+    new_var_field_offset_ptr = new_fixed_field_ptr + new_fixed_field_size;
+    old_var_field_data_ptr = old_var_field_offset_ptr + old_len_of_offsets;
+    new_var_field_data_ptr = new_var_field_offset_ptr + new_len_of_offsets;
+    curr_new_var_field_offset_ptr = new_var_field_offset_ptr;
+    curr_new_var_field_data_ptr = new_var_field_data_ptr;
+    curr_old_num_var_field = 0;
+    curr_new_num_var_field = 0;
+
+    old_null_bytes = (uchar *)old_val->data;
+    new_null_bytes = new_val_data;
+
+    
+    memcpy(&curr_old_null_pos, extra_pos, sizeof(u_int32_t));
+    extra_pos += sizeof(u_int32_t);
+    memcpy(&curr_new_null_pos, extra_pos, sizeof(u_int32_t));
+    extra_pos += sizeof(u_int32_t);
+
+    memcpy(&num_columns, extra_pos, sizeof(num_columns));
+    extra_pos += sizeof(num_columns);
+    
+    //
+    // now go through and apply the change into new_val_data
+    //
+    for (u_int32_t i = 0; i < num_columns; i++) {
+        uchar op_type = extra_pos[0];
+        bool is_null_default = false;
+        extra_pos++;
+
+        assert(op_type == COL_DROP || op_type == COL_ADD);
+        bool nullable = (extra_pos[0] != 0);
+        extra_pos++;
+        if (nullable) {
+            u_int32_t null_bit_position;
+            memcpy(&null_bit_position, extra_pos, sizeof(u_int32_t));
+            extra_pos += sizeof(u_int32_t);
+            u_int32_t num_bits;
+            if (op_type == COL_DROP) {
+                assert(curr_old_null_pos <= null_bit_position);
+                num_bits = null_bit_position - curr_old_null_pos;
+            }
+            else {
+                assert(curr_new_null_pos <= null_bit_position);
+                num_bits = null_bit_position - curr_new_null_pos;
+            }
+            copy_null_bits(
+                curr_old_null_pos,
+                curr_new_null_pos,
+                num_bits,
+                old_null_bytes,
+                new_null_bytes
+                );
+            // update the positions
+            curr_new_null_pos += num_bits;
+            curr_old_null_pos += num_bits;
+            if (op_type == COL_DROP) {
+                curr_old_null_pos++; // account for dropped column
+            }
+            else {
+                is_null_default = (extra_pos[0] != 0);
+                extra_pos++;
+                set_overall_null_position(
+                    new_null_bytes,
+                    null_bit_position,
+                    is_null_default
+                    );
+                curr_new_null_pos++; //account for added column
+            }
+        }
+        uchar col_type = extra_pos[0];
+        extra_pos++;
+        if (col_type == COL_FIXED) {
+            u_int32_t col_offset;
+            u_int32_t col_size;
+            u_int32_t num_bytes_to_copy;
+            memcpy(&col_offset, extra_pos, sizeof(u_int32_t));
+            extra_pos += sizeof(u_int32_t);
+            memcpy(&col_size, extra_pos, sizeof(u_int32_t));
+            extra_pos += sizeof(u_int32_t);
+
+            if (op_type == COL_DROP) {
+                num_bytes_to_copy = col_offset - curr_old_fixed_offset;
+            }
+            else {
+                num_bytes_to_copy = col_offset - curr_new_fixed_offset;
+            }
+            memcpy(
+                new_fixed_field_ptr + curr_new_fixed_offset,
+                old_fixed_field_ptr + curr_old_fixed_offset, 
+                num_bytes_to_copy
+                );
+            curr_old_fixed_offset += num_bytes_to_copy;
+            curr_new_fixed_offset += num_bytes_to_copy;
+            if (op_type == COL_DROP) {
+                // move old_fixed_offset val to skip OVER column that is being dropped
+                curr_old_fixed_offset += col_size;
+            }
+            else {
+                if (is_null_default) {
+                    // copy zeroes
+                    memset(new_fixed_field_ptr + curr_new_fixed_offset, 0, col_size);
+                }
+                else {
+                    // copy data from extra_pos into new row
+                    memcpy(
+                        new_fixed_field_ptr + curr_new_fixed_offset,
+                        extra_pos,
+                        col_size
+                        );
+                    extra_pos += col_size;
+                }
+                curr_new_fixed_offset += col_size;
+            }
+            
+        }
+        else if (col_type == COL_VAR) {
+            u_int32_t var_col_index;
+            memcpy(&var_col_index, extra_pos, sizeof(u_int32_t));
+            extra_pos += sizeof(u_int32_t);
+            if (op_type == COL_DROP) {
+                num_var_fields_to_copy = var_col_index - curr_old_num_var_field;
+            }
+            else {
+                num_var_fields_to_copy = var_col_index - curr_new_num_var_field;
+            }
+            copy_var_fields(
+                curr_old_num_var_field,
+                num_var_fields_to_copy,
+                old_var_field_offset_ptr,
+                old_num_offset_bytes,
+                curr_new_var_field_data_ptr,
+                curr_new_var_field_offset_ptr,
+                new_var_field_data_ptr, // pointer to beginning of var fields in new row
+                old_var_field_data_ptr, // pointer to beginning of var fields in old row
+                new_num_offset_bytes, // number of offset bytes used in new row
+                &num_data_bytes_written,
+                &num_offset_bytes_written
+                );
+            curr_new_var_field_data_ptr += num_data_bytes_written;
+            curr_new_var_field_offset_ptr += num_offset_bytes_written;
+            curr_new_num_var_field += num_var_fields_to_copy;
+            curr_old_num_var_field += num_var_fields_to_copy;
+            if (op_type == COL_DROP) {
+                curr_old_num_var_field++; // skip over dropped field
+            }
+            else {
+                if (is_null_default) {
+                    curr_new_var_field_data_ptr = write_var_field(
+                        curr_new_var_field_offset_ptr,
+                        curr_new_var_field_data_ptr,
+                        new_var_field_data_ptr,
+                        NULL, //copying no data
+                        0, //copying 0 bytes
+                        new_num_offset_bytes
+                        );
+                    curr_new_var_field_offset_ptr += new_num_offset_bytes;
+                }
+                else {
+                    u_int32_t data_length;
+                    memcpy(&data_length, extra_pos, sizeof(data_length));
+                    extra_pos += sizeof(data_length);
+                    curr_new_var_field_data_ptr = write_var_field(
+                        curr_new_var_field_offset_ptr,
+                        curr_new_var_field_data_ptr,
+                        new_var_field_data_ptr,
+                        extra_pos, //copying data from mutator
+                        data_length, //copying data_length bytes
+                        new_num_offset_bytes
+                        );
+                    extra_pos += data_length;
+                    curr_new_var_field_offset_ptr += new_num_offset_bytes;
+                }
+                curr_new_num_var_field++; //account for added column
+            }
+        }
+        else if (col_type == COL_BLOB) {
+            // handle blob data later
+            continue;
+        }
+        else {
+            assert(false);
+        }
+    }
+    // finish copying the null stuff
+    old_null_bits_left = 8*old_num_null_bytes - curr_old_null_pos;
+    new_null_bits_left = 8*new_num_null_bytes - curr_new_null_pos;
+    overall_null_bits_left = old_null_bits_left;
+    set_if_smaller(overall_null_bits_left, new_null_bits_left);
+    copy_null_bits(
+        curr_old_null_pos,
+        curr_new_null_pos,
+        overall_null_bits_left,
+        old_null_bytes,
+        new_null_bytes
+        );
+    // finish copying fixed field stuff
+    num_bytes_left = old_fixed_field_size - curr_old_fixed_offset;
+    memcpy(
+        new_fixed_field_ptr + curr_new_fixed_offset,
+        old_fixed_field_ptr + curr_old_fixed_offset, 
+        num_bytes_left
+        );
+    curr_old_fixed_offset += num_bytes_left;
+    curr_new_fixed_offset += num_bytes_left;
+    // sanity check
+    assert(curr_new_fixed_offset == new_fixed_field_size);
+
+    // finish copying var field stuff
+    num_var_fields_to_copy = old_num_var_fields - curr_old_num_var_field;
+    copy_var_fields(
+        curr_old_num_var_field,
+        num_var_fields_to_copy,
+        old_var_field_offset_ptr,
+        old_num_offset_bytes,
+        curr_new_var_field_data_ptr,
+        curr_new_var_field_offset_ptr,
+        new_var_field_data_ptr, // pointer to beginning of var fields in new row
+        old_var_field_data_ptr, // pointer to beginning of var fields in old row
+        new_num_offset_bytes, // number of offset bytes used in new row
+        &num_data_bytes_written,
+        &num_offset_bytes_written
+        );
+    curr_new_var_field_offset_ptr += num_offset_bytes_written;
+    curr_new_var_field_data_ptr += num_data_bytes_written;
+    // sanity check
+    assert(curr_new_var_field_offset_ptr == new_var_field_data_ptr);
+
+    // start handling blobs
+    get_blob_field_info(
+        &start_blob_offset, 
+        old_len_of_offsets,
+        old_var_field_data_ptr,
+        old_num_offset_bytes
+        );
+    start_blob_ptr = old_var_field_data_ptr + start_blob_offset;
+    // if nothing else in extra, then there are no blobs to add or drop, so can copy blobs straight
+    if ((extra_pos - extra_pos_start) == extra->size) {
+        num_blob_bytes = old_val->size - (start_blob_ptr - old_null_bytes);
+        memcpy(curr_new_var_field_data_ptr, start_blob_ptr, num_blob_bytes);
+        curr_new_var_field_data_ptr += num_blob_bytes;
+    }
+    // else, there is blob information to process
+    else {
+        uchar* len_bytes = NULL;
+        u_int32_t curr_old_blob = 0;
+        u_int32_t curr_new_blob = 0;
+        u_int32_t num_old_blobs = 0;
+        uchar* curr_old_blob_ptr = start_blob_ptr;
+        memcpy(&num_old_blobs, extra_pos, sizeof(num_old_blobs));
+        extra_pos += sizeof(num_old_blobs);
+        len_bytes = extra_pos;
+        extra_pos += num_old_blobs;
+        // copy over blob fields one by one
+        while ((extra_pos - extra_pos_start) < extra->size) {
+            uchar op_type = extra_pos[0];
+            extra_pos++;
+            u_int32_t num_blobs_to_copy = 0;
+            u_int32_t blob_index;
+            memcpy(&blob_index, extra_pos, sizeof(blob_index));
+            extra_pos += sizeof(blob_index);
+            assert (op_type == COL_DROP || op_type == COL_ADD);
+            if (op_type == COL_DROP) {
+                num_blobs_to_copy = blob_index - curr_old_blob;
+            }
+            else {
+                num_blobs_to_copy = blob_index - curr_new_blob;
+            }
+            for (u_int32_t i = 0; i < num_blobs_to_copy; i++) {
+                u_int32_t num_bytes_written = copy_toku_blob(
+                    curr_new_var_field_data_ptr,
+                    curr_old_blob_ptr,
+                    len_bytes[curr_old_blob + i],
+                    false
+                    );
+                curr_old_blob_ptr += num_bytes_written;
+                curr_new_var_field_data_ptr += num_bytes_written;
+            }
+            curr_old_blob += num_blobs_to_copy;
+            curr_new_blob += num_blobs_to_copy;
+            if (op_type == COL_DROP) {
+                // skip over blob in row
+                u_int32_t num_bytes = copy_toku_blob(
+                    NULL,
+                    curr_old_blob_ptr,
+                    len_bytes[curr_old_blob],
+                    true
+                    );
+                curr_old_blob++;
+                curr_old_blob_ptr += num_bytes;
+            }
+            else {
+                // copy new data
+                u_int32_t new_len_bytes = extra_pos[0];
+                extra_pos++;
+                u_int32_t num_bytes = copy_toku_blob(
+                    curr_new_var_field_data_ptr,
+                    extra_pos,
+                    new_len_bytes,
+                    false
+                    );
+                curr_new_blob++;
+                curr_new_var_field_data_ptr += num_bytes;
+                extra_pos += num_bytes;
+            }                
+        }
+        num_blob_bytes = old_val->size - (curr_old_blob_ptr - old_null_bytes);
+        memcpy(curr_new_var_field_data_ptr, curr_old_blob_ptr, num_blob_bytes);
+        curr_new_var_field_data_ptr += num_blob_bytes;
+    }
+    new_val.data = new_val_data;
+    new_val.size = curr_new_var_field_data_ptr - new_val_data;
+    set_val(&new_val, set_extra);
+    
+    error = 0;
+cleanup:
+    my_free(new_val_data, MYF(MY_ALLOW_ZERO_PTR));
+    return error;    
+}
+
+#endif