bug#16771 - ndb dd - fix leak of extents

storage/ndb/src/kernel/blocks/dbtup/Dbtup.hpp

@@ -565,7 +565,6 @@ typedef Ptr<Fragoperrec> FragoperrecPtr;
      * 
      */
     STATIC_CONST( SZ = EXTENT_SEARCH_MATRIX_SIZE );
-    Uint32 m_extent_search_matrix[SZ]; // 4x4
     DLList<Extent_info>::Head m_free_extents[SZ];
     Uint32 m_total_extent_free_space_thresholds[EXTENT_SEARCH_MATRIX_ROWS];
     Uint32 m_page_free_bits_map[EXTENT_SEARCH_MATRIX_COLS];
@@ -593,6 +592,8 @@ typedef Ptr<Fragoperrec> FragoperrecPtr;
     SLList<Extent_info, Extent_list_t>::Head m_extent_list;
   };
   
+  void dump_disk_alloc(Disk_alloc_info&);
+
 struct Fragrecord {
   Uint32 nextStartRange;
   Uint32 currentPageRange;

storage/ndb/src/kernel/blocks/dbtup/DbtupDiskAlloc.cpp

@@ -17,6 +17,112 @@
 #define DBTUP_C
 #include "Dbtup.hpp"
 
+static
+NdbOut&
+operator<<(NdbOut& out, const Ptr<Dbtup::Page> & ptr)
+{
+  out << "[ Page: ptr.i: " << ptr.i 
+      << " [ m_file_no: " << ptr.p->m_file_no
+      << " m_page_no: " << ptr.p->m_page_no << "]"
+      << " list_index: " << ptr.p->list_index 
+      << " free_space: " << ptr.p->free_space
+      << " uncommitted_used_space: " << ptr.p->uncommitted_used_space
+      << " ]";
+  return out;
+}
+
+static
+NdbOut&
+operator<<(NdbOut& out, const Ptr<Dbtup::Page_request> & ptr)
+{
+  out << "[ Page_request: ptr.i: " << ptr.i
+      << " " << ptr.p->m_key
+      << " m_estimated_free_space: " << ptr.p->m_estimated_free_space
+      << " m_list_index: " << ptr.p->m_list_index
+      << " m_frag_ptr_i: " << ptr.p->m_frag_ptr_i
+      << " m_extent_info_ptr: " << ptr.p->m_extent_info_ptr
+      << " m_ref_count: " << ptr.p->m_ref_count
+      << " m_uncommitted_used_space: " << ptr.p->m_uncommitted_used_space
+      << " ]";
+  
+  return out;
+}
+
+static
+NdbOut&
+operator<<(NdbOut& out, const Ptr<Dbtup::Extent_info> & ptr)
+{
+  out << "[ Extent_info: ptr.i " << ptr.i
+      << " " << ptr.p->m_key
+      << " m_first_page_no: " << ptr.p->m_first_page_no
+      << " m_free_space: " << ptr.p->m_free_space
+      << " m_free_matrix_pos: " << ptr.p->m_free_matrix_pos
+      << " m_free_page_count: [";
+
+  for(Uint32 i = 0; i<Dbtup::EXTENT_SEARCH_MATRIX_COLS; i++)
+    out << " " << ptr.p->m_free_page_count[i];
+  out << " ] ]";
+
+  return out;
+}
+
+void 
+Dbtup::dump_disk_alloc(Dbtup::Disk_alloc_info & alloc)
+{
+  ndbout_c("dirty pages");
+  for(Uint32 i = 0; i<MAX_FREE_LIST; i++)
+  {
+    printf("  %d : ", i);
+    Ptr<Page> ptr;
+    ArrayPool<Page> *pool= (ArrayPool<Page>*)&m_global_page_pool;
+    LocalDLList<Page> list(*pool, alloc.m_dirty_pages[i]);
+    for(list.first(ptr); !ptr.isNull(); list.next(ptr))
+    {
+      ndbout << ptr << " ";
+    }
+    ndbout_c("");
+  }
+  ndbout_c("page requests");
+  for(Uint32 i = 0; i<MAX_FREE_LIST; i++)
+  {
+    printf("  %d : ", i);
+    Ptr<Page_request> ptr;
+    LocalDLList<Page_request> list(c_page_request_pool, 
+				   alloc.m_page_requests[i]);
+    for(list.first(ptr); !ptr.isNull(); list.next(ptr))
+    {
+      ndbout << ptr << " ";
+    }
+    ndbout_c("");
+  }
+
+  ndbout_c("Extent matrix");
+  for(Uint32 i = 0; i<alloc.SZ; i++)
+  {
+    printf("  %d : ", i);
+    Ptr<Extent_info> ptr;
+    LocalDLList<Extent_info> list(c_extent_pool, alloc.m_free_extents[i]);
+    for(list.first(ptr); !ptr.isNull(); list.next(ptr))
+    {
+      ndbout << ptr << " ";
+    }
+    ndbout_c("");
+  }
+
+  if (alloc.m_curr_extent_info_ptr_i != RNIL)
+  {
+    Ptr<Extent_info> ptr;
+    c_extent_pool.getPtr(ptr, alloc.m_curr_extent_info_ptr_i);
+    ndbout << "current extent: " << ptr << endl;
+  }
+}
+
+#if defined VM_TRACE || true
+#define ddassert(x) do { if(unlikely(!(x))) { dump_disk_alloc(alloc); ndbrequire(false); } } while(0)
+#else
+#define ddassert(x)
+#endif
+
 Dbtup::Disk_alloc_info::Disk_alloc_info(const Tablerec* tabPtrP, 
 					Uint32 extent_size)
 {
@@ -60,19 +166,19 @@ Dbtup::Disk_alloc_info::find_extent(Uint32 sz) const
    * Find the biggest available (with most free space)
    * Return position in matrix
    */
+  Uint32 col = calc_page_free_bits(sz);
   Uint32 mask= EXTENT_SEARCH_MATRIX_COLS - 1;
   for(Uint32 i= 0; i<EXTENT_SEARCH_MATRIX_SIZE; i++)
   {
     // Check that it can cater for request
-    if (m_extent_search_matrix[i] < sz)
+    if (!m_free_extents[i].isEmpty())
     {
-      i = (i + mask) & ~mask;
-      continue;
+      return i;
     }
     
-    if (!m_free_extents[i].isEmpty())
+    if ((i & mask) >= col)
     {
-      return i;
+      i = (i & ~mask) + mask;
     }
   }
   
@@ -92,13 +198,7 @@ Dbtup::Disk_alloc_info::calc_extent_pos(const Extent_info* extP) const
    *   if zero (or very small free space) put 
    *     absolutly last
    */
-  {
-    
-    printf("free space %d free_page_thresholds ", free);
-    for(Uint32 i = 0; i<EXTENT_SEARCH_MATRIX_ROWS; i++)
-      printf("%d ", m_total_extent_free_space_thresholds[i]);
-    ndbout_c("");
-    
+  {    
     const Uint32 *arr= m_total_extent_free_space_thresholds;
     for(; free < * arr++; row++)
       assert(row < EXTENT_SEARCH_MATRIX_ROWS);
@@ -123,11 +223,6 @@ Dbtup::Disk_alloc_info::calc_extent_pos(const Extent_info* extP) const
    */
   Uint32 pos= (row * (mask + 1)) + (col & mask);
   
-  printf("free space %d free_page_count ", free);
-  for(Uint32 i = 0; i<EXTENT_SEARCH_MATRIX_COLS; i++)
-    printf("%d ", extP->m_free_page_count[i]);
-  ndbout_c(" -> row: %d col: %d -> pos= %d", row, col, pos);
-
   assert(pos < EXTENT_SEARCH_MATRIX_SIZE);
   return pos;
 }
@@ -237,7 +332,9 @@ Dbtup::disk_page_prealloc(Signal* signal,
        *   and since it couldn't accomadate the request
        *   we put it on the free list
        */
+      alloc.m_curr_extent_info_ptr_i = RNIL;
       Uint32 pos= alloc.calc_extent_pos(ext.p);
+      ext.p->m_free_matrix_pos = pos;
       LocalDLList<Extent_info> list(c_extent_pool, alloc.m_free_extents[pos]);
       list.add(ext);
     }
@@ -290,7 +387,14 @@ Dbtup::disk_page_prealloc(Signal* signal,
     alloc.m_curr_extent_info_ptr_i= ext.i;
     ext.p->m_free_matrix_pos= RNIL;
     pageBits= tsman.alloc_page_from_extent(&ext.p->m_key, bits);
-    ndbassert(pageBits >= 0);
+#ifdef VM_TRACE
+    ddassert(pageBits >= 0);
+#else
+    if (unlikely(pageBits < 0))
+    {
+      return -AllocExtentReq::NoExtentAvailable;
+    }
+#endif
   }
   
   /**
@@ -305,18 +409,18 @@ Dbtup::disk_page_prealloc(Signal* signal,
    */
   Uint32 size= alloc.calc_page_free_space((Uint32)pageBits);
   
-  ndbassert(size >= sz);
+  ddassert(size >= sz);
   Uint32 new_size = size - sz;   // Subtract alloc rec
   req.p->m_estimated_free_space= new_size; // Store on page request
 
   Uint32 newPageBits= alloc.calc_page_free_bits(new_size);
   if (newPageBits != (Uint32)pageBits)
   {
-    ndbassert(ext.p->m_free_page_count[pageBits] > 0);
+    ddassert(ext.p->m_free_page_count[pageBits] > 0);
     ext.p->m_free_page_count[pageBits]--;
     ext.p->m_free_page_count[newPageBits]++;
   }
-  ndbassert(ext.p->m_free_space >= sz);
+  ddassert(ext.p->m_free_space >= sz);
   ext.p->m_free_space -= sz;
   
   // And put page request in correct free list
@@ -365,13 +469,13 @@ Dbtup::disk_page_prealloc_dirty_page(Disk_alloc_info & alloc,
 				     Ptr<Page> pagePtr, 
 				     Uint32 old_idx, Uint32 sz)
 {
-  ndbassert(pagePtr.p->list_index == old_idx);
+  ddassert(pagePtr.p->list_index == old_idx);
 
   Uint32 free= pagePtr.p->free_space;
   Uint32 used= pagePtr.p->uncommitted_used_space + sz;
   Uint32 ext= pagePtr.p->m_extent_info_ptr;
   
-  ndbassert(free >= used);
+  ddassert(free >= used);
   Ptr<Extent_info> extentPtr;
   c_extent_pool.getPtr(extentPtr, ext);
 
@@ -385,14 +489,14 @@ Dbtup::disk_page_prealloc_dirty_page(Disk_alloc_info & alloc,
     old_list.remove(pagePtr);
     new_list.add(pagePtr);
 
-    ndbassert(extentPtr.p->m_free_page_count[old_idx]);
+    ddassert(extentPtr.p->m_free_page_count[old_idx]);
     extentPtr.p->m_free_page_count[old_idx]--;
     extentPtr.p->m_free_page_count[new_idx]++;
     pagePtr.p->list_index= new_idx;  
   }
 
   pagePtr.p->uncommitted_used_space = used;
-  ndbassert(extentPtr.p->m_free_space >= sz);
+  ddassert(extentPtr.p->m_free_space >= sz);
   extentPtr.p->m_free_space -= sz;
   Uint32 old_pos= extentPtr.p->m_free_matrix_pos;
   if (old_pos != RNIL) // Current extent
@@ -417,7 +521,7 @@ Dbtup::disk_page_prealloc_transit_page(Disk_alloc_info& alloc,
 				       Ptr<Page_request> req, 
 				       Uint32 old_idx, Uint32 sz)
 {
-  ndbassert(req.p->m_list_index == old_idx);
+  ddassert(req.p->m_list_index == old_idx);
 
   Uint32 free= req.p->m_estimated_free_space;
   Uint32 used= req.p->m_uncommitted_used_space + sz;
@@ -426,7 +530,7 @@ Dbtup::disk_page_prealloc_transit_page(Disk_alloc_info& alloc,
   Ptr<Extent_info> extentPtr;
   c_extent_pool.getPtr(extentPtr, ext);
 
-  ndbassert(free >= sz);
+  ddassert(free >= sz);
   Uint32 new_idx= alloc.calc_page_free_bits(free - sz);
   
   if (old_idx != new_idx)
@@ -437,7 +541,7 @@ Dbtup::disk_page_prealloc_transit_page(Disk_alloc_info& alloc,
     old_list.remove(req);
     new_list.add(req);
 
-    ndbassert(extentPtr.p->m_free_page_count[old_idx]);
+    ddassert(extentPtr.p->m_free_page_count[old_idx]);
     extentPtr.p->m_free_page_count[old_idx]--;
     extentPtr.p->m_free_page_count[new_idx]++;
     req.p->m_list_index= new_idx;  
@@ -445,7 +549,7 @@ Dbtup::disk_page_prealloc_transit_page(Disk_alloc_info& alloc,
 
   req.p->m_uncommitted_used_space = used;
   req.p->m_estimated_free_space = free - sz;
-  ndbassert(extentPtr.p->m_free_space >= sz);
+  ddassert(extentPtr.p->m_free_space >= sz);
   extentPtr.p->m_free_space -= sz;
   Uint32 old_pos= extentPtr.p->m_free_matrix_pos;
   if (old_pos != RNIL) // Current extent
@@ -551,11 +655,11 @@ Dbtup::disk_page_prealloc_callback_common(Signal* signal,
    * 3) register callback in pgman (unmap callback)
    * 4) inform pgman about current users
    */
-  ndbassert((page->list_index & 0x8000) == 0x8000);
-  ndbassert(page->m_extent_info_ptr == req.p->m_extent_info_ptr);
-  ndbassert(page->m_page_no == req.p->m_key.m_page_no);
-  ndbassert(page->m_file_no == req.p->m_key.m_file_no);
   Disk_alloc_info& alloc= fragPtr.p->m_disk_alloc_info;
+  ddassert((page->list_index & 0x8000) == 0x8000);
+  ddassert(page->m_extent_info_ptr == req.p->m_extent_info_ptr);
+  ddassert(page->m_page_no == req.p->m_key.m_page_no);
+  ddassert(page->m_file_no == req.p->m_key.m_file_no);
   
   Uint32 old_idx = req.p->m_list_index;
   Uint32 free= req.p->m_estimated_free_space;
@@ -564,9 +668,9 @@ Dbtup::disk_page_prealloc_callback_common(Signal* signal,
   Uint32 real_free = page->free_space;
   Uint32 real_used = used + page->uncommitted_used_space;
  
-  ndbassert(real_free >= free);
-  ndbassert(real_free >= real_used);
-  ndbassert(alloc.calc_page_free_bits(free) == old_idx);
+  ddassert(real_free >= free);
+  ddassert(real_free >= real_used);
+  ddassert(alloc.calc_page_free_bits(free) == old_idx);
   Uint32 new_idx= alloc.calc_page_free_bits(real_free - real_used);
   
   /**
@@ -587,7 +691,7 @@ Dbtup::disk_page_prealloc_callback_common(Signal* signal,
     
     if (old_idx != new_idx)
     {
-      ndbassert(extentPtr.p->m_free_page_count[old_idx]);
+      ddassert(extentPtr.p->m_free_page_count[old_idx]);
       extentPtr.p->m_free_page_count[old_idx]--;
       extentPtr.p->m_free_page_count[new_idx]++;
     }
@@ -721,13 +825,14 @@ Dbtup::disk_page_alloc(Signal* signal,
 		       Local_key* key, PagePtr pagePtr, Uint32 gci)
 {
   Uint32 logfile_group_id= fragPtrP->m_logfile_group_id;
+  Disk_alloc_info& alloc= fragPtrP->m_disk_alloc_info;
 
   Uint64 lsn;
   Uint32 old_free = pagePtr.p->free_space;
-  Uint32 old_bits= fragPtrP->m_disk_alloc_info.calc_page_free_bits(old_free);
+  Uint32 old_bits= alloc.calc_page_free_bits(old_free);
   if (tabPtrP->m_attributes[DD].m_no_of_varsize == 0)
   {
-    ndbassert(pagePtr.p->uncommitted_used_space > 0);
+    ddassert(pagePtr.p->uncommitted_used_space > 0);
     pagePtr.p->uncommitted_used_space--;
     key->m_page_idx= ((Fix_page*)pagePtr.p)->alloc_record();
     lsn= disk_page_undo_alloc(pagePtr.p, key, 1, gci, logfile_group_id);
@@ -735,7 +840,7 @@ Dbtup::disk_page_alloc(Signal* signal,
   else
   {
     Uint32 sz= key->m_page_idx;
-    ndbassert(pagePtr.p->uncommitted_used_space >= sz);
+    ddassert(pagePtr.p->uncommitted_used_space >= sz);
     pagePtr.p->uncommitted_used_space -= sz;
     key->m_page_idx= ((Var_page*)pagePtr.p)->
       alloc_record(sz, (Var_page*)ctemp_page, 0);
@@ -744,7 +849,7 @@ Dbtup::disk_page_alloc(Signal* signal,
   }
 
   Uint32 new_free = pagePtr.p->free_space;
-  Uint32 new_bits= fragPtrP->m_disk_alloc_info.calc_page_free_bits(new_free);
+  Uint32 new_bits= alloc.calc_page_free_bits(new_free);
   
   if (old_bits != new_bits)
   {
@@ -806,20 +911,20 @@ Dbtup::disk_page_free(Signal *signal,
 
   Uint32 ext = pagePtr.p->m_extent_info_ptr;
   Uint32 used = pagePtr.p->uncommitted_used_space;
-  ndbassert(old_free >= used);
-  ndbassert(new_free >= used);
-  ndbassert(new_free >= old_free);
+  ddassert(old_free >= used);
+  ddassert(new_free >= used);
+  ddassert(new_free >= old_free);
   page_idx = pagePtr.p->list_index;
   Uint32 old_idx = page_idx & 0x7FFF;
   Uint32 new_idx = alloc.calc_page_free_bits(new_free - used);
-  ndbassert(alloc.calc_page_free_bits(old_free - used) == old_idx);
+  ddassert(alloc.calc_page_free_bits(old_free - used) == old_idx);
 
   Ptr<Extent_info> extentPtr;
   c_extent_pool.getPtr(extentPtr, ext);
 
   if (old_idx != new_idx)
   {
-    ndbassert(extentPtr.p->m_free_page_count[old_idx]);
+    ddassert(extentPtr.p->m_free_page_count[old_idx]);
     extentPtr.p->m_free_page_count[old_idx]--;
     extentPtr.p->m_free_page_count[new_idx]++;
 
@@ -915,16 +1020,16 @@ Dbtup::disk_page_abort_prealloc_callback_1(Signal* signal,
   Uint32 ext = pagePtr.p->m_extent_info_ptr;
 
   Uint32 old_idx = page_idx & 0x7FFF;
-  ndbassert(free >= used);
-  ndbassert(used >= sz);
-  ndbassert(alloc.calc_page_free_bits(free - used) == old_idx);
+  ddassert(free >= used);
+  ddassert(used >= sz);
+  ddassert(alloc.calc_page_free_bits(free - used) == old_idx);
   Uint32 new_idx = alloc.calc_page_free_bits(free - used + sz);
 
   Ptr<Extent_info> extentPtr;
   c_extent_pool.getPtr(extentPtr, ext);
   if (old_idx != new_idx)
   {
-    ndbassert(extentPtr.p->m_free_page_count[old_idx]);
+    ddassert(extentPtr.p->m_free_page_count[old_idx]);
     extentPtr.p->m_free_page_count[old_idx]--;
     extentPtr.p->m_free_page_count[new_idx]++;
 

storage/ndb/src/kernel/blocks/pgman.cpp

@@ -40,6 +40,12 @@
 #define dbg(x)
 #endif
 
+#if 1
+#define DBG_LCP(x)
+#else
+#define DBG_LCP(x) ndbout << x
+#endif
+
 Pgman::Pgman(const Configuration & conf) :
   SimulatedBlock(PGMAN, conf),
   m_file_map(m_data_buffer_pool),
@@ -1083,6 +1089,7 @@ Pgman::execLCP_FRAG_ORD(Signal* signal)
   LcpFragOrd* ord = (LcpFragOrd*)signal->getDataPtr();
   ndbrequire(ord->lcpId >= m_last_lcp_complete + 1 || m_last_lcp_complete == 0);
   m_last_lcp = ord->lcpId;
+  DBG_LCP("execLCP_FRAG_ORD" << endl);
 
   ndbrequire(!m_lcp_outstanding);
   ndbrequire(m_lcp_copy_page_free);
@@ -1104,6 +1111,8 @@ Pgman::execEND_LCP_REQ(Signal* signal)
   EndLcpReq* req = (EndLcpReq*)signal->getDataPtr();
   m_end_lcp_req = *req;
 
+  DBG_LCP("execEND_LCP_REQ" << endl);
+
 #ifdef VM_TRACE
   debugOut
     << "PGMAN: execEND_LCP_REQ"
@@ -1117,6 +1126,7 @@ Pgman::execEND_LCP_REQ(Signal* signal)
     ndbrequire(! m_lcp_loop_on);
     signal->theData[0] = m_end_lcp_req.senderData;
     sendSignal(m_end_lcp_req.senderRef, GSN_END_LCP_CONF, signal, 1, JBB);
+    DBG_LCP("GSN_END_LCP_CONF" << endl);
   }
 
   m_last_lcp_complete = m_last_lcp;
@@ -1149,6 +1159,8 @@ Pgman::process_lcp(Signal* signal)
       Ptr<Page_entry>& ptr = iter.curr;
       Uint16 state = ptr.p->m_state;
 
+      DBG_LCP("PROCESS LCP: " << ptr);
+      
       if (ptr.p->m_last_lcp < m_last_lcp &&
           (state & Page_entry::DIRTY))
       {
@@ -1159,6 +1171,7 @@ Pgman::process_lcp(Signal* signal)
         }
         if (state & Page_entry::BUSY)
         {
+	  DBG_LCP(" BUSY" << endl);
           break;  // wait for it
         }
         if (state & Page_entry::LOCKED)
@@ -1169,6 +1182,7 @@ Pgman::process_lcp(Signal* signal)
            */
           if (!m_lcp_copy_page_free)
           {
+	    DBG_LCP(" !m_lcp_copy_page_free" << endl);
             break;
           }
           m_lcp_copy_page_free = false;
@@ -1183,10 +1197,12 @@ Pgman::process_lcp(Signal* signal)
         }
         else if (state & Page_entry::PAGEOUT)
         {
+	  DBG_LCP(" PAGEOUT -> state |= LCP" << endl);
           set_page_state(ptr, state | Page_entry::LCP);
         }
         else
         {
+	  DBG_LCP(" pageout()" << endl);
           ptr.p->m_state |= Page_entry::LCP;
           pageout(signal, ptr);
         }
@@ -1205,11 +1221,15 @@ Pgman::process_lcp(Signal* signal)
     {
       signal->theData[0] = m_end_lcp_req.senderData;
       sendSignal(m_end_lcp_req.senderRef, GSN_END_LCP_CONF, signal, 1, JBB);
+      DBG_LCP("GSN_END_LCP_CONF" << endl);
     }
+    DBG_LCP(" -- RETURN FALSE" << endl);
     m_last_lcp_complete = m_last_lcp;
     m_lcp_curr_bucket = ~(Uint32)0;
     return false;
   }
+
+  DBG_LCP(" -- RETURN TRUE" << endl);
   return true;
 }