Fix crash in BDB from improper cleanup of transactions, and avoid problem

in NDB that the fix would cause due to improper registration of a transaction
that isn't meant to be seen by the handler layer. (Bug #14212)

Thanks to Sergei Golubchik for helping with this.


mysql-test/r/bdb.result:
  Add new result
mysql-test/t/bdb.test:
  Add new regression test
sql/ha_ndbcluster.cc:
  Don't register transaction if transactions are "disabled".
sql/handler.cc:
  Commit statement before trying to end transaction.

mysql-test/r/bdb.result

@@ -1895,3 +1895,8 @@ t1	CREATE TABLE `t1` (
 ) ENGINE=BerkeleyDB DEFAULT CHARSET=latin1
 drop table t1;
 set storage_engine=MyISAM;
+set autocommit=0;
+create table t1 (a int) engine=bdb;
+commit;
+alter table t1 add primary key(a);
+drop table t1;

mysql-test/t/bdb.test

@@ -973,3 +973,14 @@ drop table t1;
 
 # End varchar test
 eval set storage_engine=$default;
+
+#
+# Bug #14212: Server crash after COMMIT + ALTER TABLE
+#
+set autocommit=0;
+create table t1 (a int) engine=bdb;
+commit;
+alter table t1 add primary key(a);
+drop table t1;
+
+# End of 5.0 tests

sql/ha_ndbcluster.cc

@@ -3221,6 +3221,10 @@ int ha_ndbcluster::external_lock(THD *thd, int lock_type)
   if (lock_type != F_UNLCK)
   {
     DBUG_PRINT("info", ("lock_type != F_UNLCK"));
+    if (!thd->transaction.on)
+      m_transaction_on= FALSE;
+    else
+      m_transaction_on= thd->variables.ndb_use_transactions;
     if (!thd_ndb->lock_count++)
     {
       PRINT_OPTION_FLAGS(thd);
@@ -3235,7 +3239,8 @@ int ha_ndbcluster::external_lock(THD *thd, int lock_type)
           ERR_RETURN(ndb->getNdbError());
         no_uncommitted_rows_reset(thd);
         thd_ndb->stmt= trans;
-        trans_register_ha(thd, FALSE, &ndbcluster_hton);
+        if (m_transaction_on)
+          trans_register_ha(thd, FALSE, &ndbcluster_hton);
       } 
       else 
       { 
@@ -3250,7 +3255,8 @@ int ha_ndbcluster::external_lock(THD *thd, int lock_type)
             ERR_RETURN(ndb->getNdbError());
           no_uncommitted_rows_reset(thd);
           thd_ndb->all= trans; 
-          trans_register_ha(thd, TRUE, &ndbcluster_hton);
+          if (m_transaction_on)
+            trans_register_ha(thd, TRUE, &ndbcluster_hton);
 
           /*
             If this is the start of a LOCK TABLE, a table look 
@@ -3284,10 +3290,6 @@ int ha_ndbcluster::external_lock(THD *thd, int lock_type)
     m_ha_not_exact_count= !thd->variables.ndb_use_exact_count;
     m_autoincrement_prefetch= 
       (ha_rows) thd->variables.ndb_autoincrement_prefetch_sz;
-    if (!thd->transaction.on)
-      m_transaction_on= FALSE;
-    else
-      m_transaction_on= thd->variables.ndb_use_transactions;
 
     m_active_trans= thd_ndb->all ? thd_ndb->all : thd_ndb->stmt;
     DBUG_ASSERT(m_active_trans);

sql/handler.cc

@@ -1910,7 +1910,8 @@ int ha_enable_transaction(THD *thd, bool on)
       is an optimization hint that storage engine is free to ignore.
       So, let's commit an open transaction (if any) now.
     */
-    error= end_trans(thd, COMMIT);
+    if (!(error= ha_commit_stmt(thd)))
+      error= end_trans(thd, COMMIT);
   }
   DBUG_RETURN(error);
 }